From 91940d9851f0605d304ef8b2170068b7ccbae342 Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Fri, 10 May 2024 16:13:10 -0300 Subject: [PATCH 1/7] Refactoring Allocator classes --- .../main/java/com/cloud/host/dao/HostDao.java | 2 +- .../java/com/cloud/host/dao/HostDaoImpl.java | 4 +- .../allocator/impl/RandomAllocator.java | 2 +- .../allocator/impl/FirstFitAllocator.java | 524 ++++++++++-------- 4 files changed, 287 insertions(+), 245 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index 08380ed8b405..05d6c4d3d280 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -166,7 +166,7 @@ public interface HostDao extends GenericDao, StateDao listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType); - List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags); + List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags); List findClustersThatMatchHostTagRule(String computeOfferingTags); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 170c6a45fc3a..77e2516f3aa3 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -1350,7 +1350,7 @@ private List findHostByComputeOfferings(String computeOfferingTags){ } } - public List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags) { + public List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags) { List hostTagVOList = _hostTagsDao.findHostRuleTags(); List result = new ArrayList<>(); for (HostTagVO rule: hostTagVOList) { @@ -1364,7 +1364,7 @@ public List findHostsWithTagRuleThatMatchComputeOferringTags(String comp public List findClustersThatMatchHostTagRule(String computeOfferingTags) { Set result = new HashSet<>(); - List hosts = findHostsWithTagRuleThatMatchComputeOferringTags(computeOfferingTags); + List hosts = findHostsWithTagRuleThatMatchComputeOfferingTags(computeOfferingTags); for (HostVO host: hosts) { result.add(host.getClusterId()); } diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java index a71ae26e670d..974e0eb726c9 100644 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -120,7 +120,7 @@ private List findSuitableHosts(VirtualMachineProfile vmProfile, Deployment hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); } } - hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(offeringHostTag)); + hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOfferingTags(hostTag)); if (hostsCopy.isEmpty()) { logger.info("No suitable host found for VM [{}] in {}.", vmProfile, hostTag); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 99ac2492e833..c36c26f67b67 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -25,9 +25,12 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import com.cloud.agent.manager.allocator.HostAllocator; @@ -108,6 +111,9 @@ public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan pla @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { + if (type == Host.Type.Storage) { + return null; + } long dcId = plan.getDataCenterId(); Long podId = plan.getPodId(); @@ -116,285 +122,283 @@ public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan pla VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); Account account = vmProfile.getOwner(); - boolean isVMDeployedWithUefi = false; - UserVmDetailVO userVmDetailVO = _userVmDetailsDao.findDetail(vmProfile.getId(), "UEFI"); - if(userVmDetailVO != null){ - if ("secure".equalsIgnoreCase(userVmDetailVO.getValue()) || "legacy".equalsIgnoreCase(userVmDetailVO.getValue())) { - isVMDeployedWithUefi = true; - } - } - logger.info(" Guest VM is requested with Custom[UEFI] Boot Type "+ isVMDeployedWithUefi); + String hostTagOnOffering = offering.getHostTag(); + String hostTagOnTemplate = template.getTemplateTag(); + String paramAsStringToLog = String.format("zone [%s], pod [%s], cluster [%s]", dcId, podId, clusterId); + List suitableHosts = retrieveHosts(vmProfile, type, clusterId, podId, dcId, hostTagOnOffering, hostTagOnTemplate); - if (type == Host.Type.Storage) { - // FirstFitAllocator should be used for user VMs only since it won't care whether the host is capable of routing or not - return new ArrayList<>(); + if (isSuitableHostsEmpty(vmProfile, suitableHosts, paramAsStringToLog)) { + return null; } - logger.debug("Looking for hosts in zone [{}], pod [{}], cluster [{}]", dcId, podId, clusterId); + addHostsToAvoidSet(type, avoid, clusterId, podId, dcId, suitableHosts); - String hostTagOnOffering = offering.getHostTag(); - String hostTagOnTemplate = template.getTemplateTag(); - String hostTagUefi = "UEFI"; - - boolean hasSvcOfferingTag = hostTagOnOffering != null ? true : false; - boolean hasTemplateTag = hostTagOnTemplate != null ? true : false; + return allocateTo(plan, offering, template, avoid, suitableHosts, returnUpTo, considerReservedCapacity, account); + } - List clusterHosts = new ArrayList<>(); - List hostsMatchingUefiTag = new ArrayList<>(); - if(isVMDeployedWithUefi){ - hostsMatchingUefiTag = _hostDao.listByHostCapability(type, clusterId, podId, dcId, Host.HOST_UEFI_ENABLE); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagUefi + "' are:" + hostsMatchingUefiTag); - } + private boolean isSuitableHostsEmpty(VirtualMachineProfile vmProfile, List suitableHosts, String paramAsStringToLog) { + if (suitableHosts.isEmpty()) { + logger.info("No suitable host found for VM [{}] in {}.", vmProfile, paramAsStringToLog); + return true; } + return false; + } + private List retrieveHosts(VirtualMachineProfile vmProfile, Type type, Long clusterId, Long podId, long dcId, String hostTagOnOffering, String hostTagOnTemplate) { + String haVmTag = (String) vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); + List clusterHosts; - String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); if (haVmTag != null) { clusterHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag); + } else if (ObjectUtils.allNull(hostTagOnOffering, hostTagOnTemplate)) { + clusterHosts = _resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId); } else { - if (hostTagOnOffering == null && hostTagOnTemplate == null) { - clusterHosts = _resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId); - } else { - List hostsMatchingOfferingTag = new ArrayList<>(); - List hostsMatchingTemplateTag = new ArrayList<>(); - if (hasSvcOfferingTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on SvcOffering:" + hostTagOnOffering); - } - hostsMatchingOfferingTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnOffering + "' are:" + hostsMatchingOfferingTag); - } - } - if (hasTemplateTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on Template:" + hostTagOnTemplate); - } - hostsMatchingTemplateTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnTemplate + "' are:" + hostsMatchingTemplateTag); - } - } + clusterHosts = retrieveHostsMatchingServiceOfferingAndTemplateTags(hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); + } - if (hasSvcOfferingTag && hasTemplateTag) { - hostsMatchingOfferingTag.retainAll(hostsMatchingTemplateTag); - if (logger.isDebugEnabled()) { - logger.debug("Found " + hostsMatchingOfferingTag.size() + " Hosts satisfying both tags, host ids are:" + hostsMatchingOfferingTag); - } + filterHostsWithUefiEnabled(type, vmProfile, clusterId, podId, dcId, clusterHosts); - clusterHosts = hostsMatchingOfferingTag; - } else { - if (hasSvcOfferingTag) { - clusterHosts = hostsMatchingOfferingTag; - } else { - clusterHosts = hostsMatchingTemplateTag; - } - } - } - } + addHostsBasedOnTagRules(hostTagOnOffering, clusterHosts); + + return clusterHosts; + } + + private void addHostsBasedOnTagRules(String hostTagOnOffering, List clusterHosts) { + List hostsWithTagRules = _hostDao.findHostsWithTagRuleThatMatchComputeOfferingTags(hostTagOnOffering); - if (isVMDeployedWithUefi) { - clusterHosts.retainAll(hostsMatchingUefiTag); + if (CollectionUtils.isEmpty(hostsWithTagRules)) { + logger.info("No hosts found with tag rules matching the compute offering tag {}.", hostTagOnOffering); } - clusterHosts.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); + logger.info("Found hosts {} with tag rules matching the compute offering tag [{}].", hostsWithTagRules, hostTagOnOffering); + clusterHosts.addAll(hostsWithTagRules); + } + + private List retrieveHostsMatchingServiceOfferingAndTemplateTags(String hostTagOnTemplate, String hostTagOnOffering, Type type, Long clusterId, Long podId, long dcId) { + boolean hasSvcOfferingTag = hostTagOnOffering != null; + boolean hasTemplateTag = hostTagOnTemplate != null; + List clusterHosts; + List hostsMatchingOfferingTag = new ArrayList<>(); + List hostsMatchingTemplateTag = new ArrayList<>(); + if (hasSvcOfferingTag) { + logger.debug("Looking for hosts having the tag [{}] specified in the Service Offering.", hostTagOnOffering); + hostsMatchingOfferingTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); + logger.debug("Hosts with Service Offering tag [{}] are {}.", hostTagOnOffering, hostsMatchingOfferingTag); + } - if (clusterHosts.isEmpty()) { - logger.error("No suitable host found for vm [{}] with tags [{}].", vmProfile, hostTagOnOffering); - throw new CloudRuntimeException(String.format("No suitable host found for vm [%s].", vmProfile)); + if (hasTemplateTag) { + logger.debug("Looking for hosts having the tag [{}] specified in the Template.", hostTagOnTemplate); + hostsMatchingTemplateTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + logger.debug("Hosts with Template tag [{}] are {}.", hostTagOnTemplate, hostsMatchingTemplateTag); } - // add all hosts that we are not considering to the avoid list - List allhostsInCluster = _hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); - allhostsInCluster.removeAll(clusterHosts); - logger.debug(() -> String.format("Adding hosts [%s] to the avoid set because these hosts do not support HA.", - ReflectionToStringBuilderUtils.reflectOnlySelectedFields(allhostsInCluster, "uuid", "name"))); + if (hasSvcOfferingTag && hasTemplateTag) { + hostsMatchingOfferingTag.retainAll(hostsMatchingTemplateTag); + logger.debug("Found {} Hosts satisfying both tags; host IDs are {}.", hostsMatchingOfferingTag.size(), hostsMatchingOfferingTag); + clusterHosts = hostsMatchingOfferingTag; + } else if (hasSvcOfferingTag) { + clusterHosts = hostsMatchingOfferingTag; + } else { + clusterHosts = hostsMatchingTemplateTag; + } - for (HostVO host : allhostsInCluster) { + return clusterHosts; + } + + /** + * Add all hosts to the avoid set that were not considered during the allocation + */ + private void addHostsToAvoidSet(Type type, ExcludeList avoid, Long clusterId, Long podId, long dcId, List suitableHosts) { + List allHostsInCluster = _hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); + + allHostsInCluster.removeAll(suitableHosts); + + logger.debug("Adding hosts [{}] to the avoid set because these hosts were not considered for allocation.", + () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(allHostsInCluster, "uuid", "name")); + + for (HostVO host : allHostsInCluster) { avoid.addHost(host.getId()); } + } + + private void filterHostsWithUefiEnabled(Type type, VirtualMachineProfile vmProfile, Long clusterId, Long podId, long dcId, List clusterHosts) { + UserVmDetailVO userVmDetailVO = _userVmDetailsDao.findDetail(vmProfile.getId(), "UEFI"); + + if (userVmDetailVO == null) { + return; + } + + if (!StringUtils.equalsAnyIgnoreCase(userVmDetailVO.getValue(), ApiConstants.BootMode.SECURE.toString(), ApiConstants.BootMode.LEGACY.toString())) { + return; + } - return allocateTo(plan, offering, template, avoid, clusterHosts, returnUpTo, considerReservedCapacity, account); + logger.info("Guest VM is requested with Custom[UEFI] Boot Type enabled."); + + List hostsMatchingUefiTag = _hostDao.listByHostCapability(type, clusterId, podId, dcId, Host.HOST_UEFI_ENABLE); + + logger.debug("Hosts with UEFI enabled are {}.", hostsMatchingUefiTag); + clusterHosts.retainAll(hostsMatchingUefiTag); } @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { + boolean considerReservedCapacity) { + if (type == Host.Type.Storage) { + return null; + } + long dcId = plan.getDataCenterId(); Long podId = plan.getPodId(); Long clusterId = plan.getClusterId(); ServiceOffering offering = vmProfile.getServiceOffering(); - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); + VMTemplateVO template = (VMTemplateVO) vmProfile.getTemplate(); Account account = vmProfile.getOwner(); - List suitableHosts = new ArrayList<>(); - List hostsCopy = new ArrayList<>(hosts); - - if (type == Host.Type.Storage) { - // FirstFitAllocator should be used for user VMs only since it won't care whether the host is capable of - // routing or not. - return suitableHosts; - } String hostTagOnOffering = offering.getHostTag(); String hostTagOnTemplate = template.getTemplateTag(); - boolean hasSvcOfferingTag = hostTagOnOffering != null ? true : false; - boolean hasTemplateTag = hostTagOnTemplate != null ? true : false; + List suitableHosts = (List) new ArrayList<>(hosts); - String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); - if (haVmTag != null) { - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); - } else { - if (hostTagOnOffering == null && hostTagOnTemplate == null) { - hostsCopy.retainAll(_resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId)); - } else { - if (hasSvcOfferingTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on SvcOffering:" + hostTagOnOffering); - } - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering)); - - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnOffering + "' are:" + hostsCopy); - } - } + String paramAsStringToLog = String.format("zone [%s], pod [%s], cluster [%s]", dcId, podId, clusterId); + logger.debug("Looking for hosts in {}.", paramAsStringToLog); - if (hasTemplateTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on Template:" + hostTagOnTemplate); - } + retainHostsMatchingCriteria(vmProfile, type, suitableHosts, clusterId, podId, dcId, hostTagOnOffering, hostTagOnTemplate); - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate)); + addHostsBasedOnTagRules(hostTagOnOffering, suitableHosts); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnTemplate + "' are:" + hostsCopy); - } - } - } + if (isSuitableHostsEmpty(vmProfile, suitableHosts, paramAsStringToLog)) { + return null; } - hostsCopy.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); + return allocateTo(plan, offering, template, avoid, suitableHosts, returnUpTo, considerReservedCapacity, account); + } - if (!hostsCopy.isEmpty()) { - suitableHosts = allocateTo(plan, offering, template, avoid, hostsCopy, returnUpTo, considerReservedCapacity, account); - } + private void retainHostsMatchingCriteria(VirtualMachineProfile vmProfile, Type type, List suitableHosts, Long clusterId, Long podId, long dcId, + String hostTagOnOffering, String hostTagOnTemplate) { + String haVmTag = (String) vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); + boolean hasSvcOfferingTag = hostTagOnOffering != null; + boolean hasTemplateTag = hostTagOnTemplate != null; - return suitableHosts; + if (haVmTag != null) { + suitableHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); + } else if (ObjectUtils.allNull(hostTagOnOffering, hostTagOnTemplate)) { + suitableHosts.retainAll(_resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId)); + } else { + if (hasSvcOfferingTag) { + logger.debug("Looking for hosts having the tag [{}] specified in the Service Offering.", hostTagOnOffering); + suitableHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering)); + logger.debug("Hosts with Service Offering tag [{}] are {}.", hostTagOnOffering, suitableHosts); + } + + if (hasTemplateTag) { + logger.debug("Looking for hosts having the tag [{}] specified in the Template.", hostTagOnTemplate); + suitableHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate)); + logger.debug("Hosts with Template tag [{}] are {}.", hostTagOnTemplate, suitableHosts); + } + } } protected List allocateTo(DeploymentPlan plan, ServiceOffering offering, VMTemplateVO template, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity, Account account) { - if (_allocationAlgorithm.equals("random") || _allocationAlgorithm.equals("userconcentratedpod_random")) { - // Shuffle this so that we don't check the hosts in the same order. - Collections.shuffle(hosts); - } else if (_allocationAlgorithm.equals("userdispersing")) { - hosts = reorderHostsByNumberOfVms(plan, hosts, account); - }else if(_allocationAlgorithm.equals("firstfitleastconsumed")){ - hosts = reorderHostsByCapacity(plan, hosts); + boolean considerReservedCapacity, Account account) { + switch (_allocationAlgorithm) { + case "random": + case "userconcentratedpod_random": + // Shuffle this so that we don't check the hosts in the same order. + Collections.shuffle(hosts); + break; + case "userdispersing": + hosts = reorderHostsByNumberOfVms(plan, hosts, account); + break; + case "firstfitleastconsumed": + hosts = reorderHostsByCapacity(plan, hosts); + break; } - if (logger.isDebugEnabled()) { - logger.debug("FirstFitAllocator has " + hosts.size() + " hosts to check for allocation: " + hosts); - } + logger.debug("FirstFitAllocator has {} hosts to check for allocation: {}.", hosts.size(), hosts); - // We will try to reorder the host lists such that we give priority to hosts that have - // the minimums to support a VM's requirements hosts = prioritizeHosts(template, offering, hosts); - if (logger.isDebugEnabled()) { - logger.debug("Found " + hosts.size() + " hosts for allocation after prioritization: " + hosts); - } + logger.debug("Found {} hosts for allocation after prioritization: {}.", hosts.size(), hosts); + logger.debug("Looking for frequency {} MHz and RAM {} MB.", () -> offering.getCpu() * offering.getSpeed(), offering::getRamSize); - if (logger.isDebugEnabled()) { - logger.debug("Looking for speed=" + (offering.getCpu() * offering.getSpeed()) + "Mhz, Ram=" + offering.getRamSize() + " MB"); - } + List suitableHosts = checkHostsCompatibilities(offering, avoid, hosts, returnUpTo, considerReservedCapacity); - long serviceOfferingId = offering.getId(); + logger.debug("Host Allocator returning {} suitable hosts", suitableHosts.size()); + + return suitableHosts; + } + + private List checkHostsCompatibilities(ServiceOffering offering, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { List suitableHosts = new ArrayList<>(); - ServiceOfferingDetailsVO offeringDetails = null; for (Host host : hosts) { if (suitableHosts.size() == returnUpTo) { break; } + if (avoid.shouldAvoid(host)) { - if (logger.isDebugEnabled()) { - logger.debug("Host name: " + host.getName() + ", hostId: " + host.getId() + " is in avoid set, skipping this and trying other available hosts"); - } + logger.debug("Host [{}] is in avoid set, skipping this and trying other available hosts", + () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); continue; } - //find number of guest VMs occupying capacity on this host. if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)) { - logger.debug(() -> String.format("Adding host [%s] to the avoid set because this host already has the max number of running (user and/or system) VMs.", - ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name"))); + logger.debug("Adding host [{}] to the avoid set because this host already has the max number of running (user and/or system) VMs.", + () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); avoid.addHost(host.getId()); continue; } - // Check if GPU device is required by offering and host has the availability - if ((offeringDetails = _serviceOfferingDetailsDao.findDetail(serviceOfferingId, GPU.Keys.vgpuType.toString())) != null) { - ServiceOfferingDetailsVO groupName = _serviceOfferingDetailsDao.findDetail(serviceOfferingId, GPU.Keys.pciDevice.toString()); - if(!_resourceMgr.isGPUDeviceAvailable(host.getId(), groupName.getValue(), offeringDetails.getValue())){ - logger.debug(String.format("Adding host [%s] to avoid set, because this host does not have required GPU devices available.", - ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name"))); - avoid.addHost(host.getId()); - continue; - } + if (offeringRequestedVGpuAndHostDoesNotHaveIt(offering, avoid, host)) { + continue; } + Pair cpuCapabilityAndCapacity = _capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); if (cpuCapabilityAndCapacity.first() && cpuCapabilityAndCapacity.second()) { - if (logger.isDebugEnabled()) { - logger.debug("Found a suitable host, adding to list: " + host.getId()); - } + logger.debug("Found a suitable host, adding to list host [{}].", () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); suitableHosts.add(host); } else { - if (logger.isDebugEnabled()) { - logger.debug("Not using host " + host.getId() + "; host has cpu capability? " + cpuCapabilityAndCapacity.first() + ", host has capacity?" + cpuCapabilityAndCapacity.second()); - } + logger.debug("Not using host {}; host has cpu capability? {}, host has capacity? {}.", + () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name"), cpuCapabilityAndCapacity::first, cpuCapabilityAndCapacity::second); avoid.addHost(host.getId()); } } + return suitableHosts; + } + - if (logger.isDebugEnabled()) { - logger.debug("Host Allocator returning " + suitableHosts.size() + " suitable hosts"); + private boolean offeringRequestedVGpuAndHostDoesNotHaveIt(ServiceOffering offering, ExcludeList avoid, Host host) { + long serviceOfferingId = offering.getId(); + ServiceOfferingDetailsVO requestedVGpuType = _serviceOfferingDetailsDao.findDetail(serviceOfferingId, GPU.Keys.vgpuType.toString()); + + if (requestedVGpuType == null) { + return false; } - return suitableHosts; + ServiceOfferingDetailsVO groupName = _serviceOfferingDetailsDao.findDetail(serviceOfferingId, GPU.Keys.pciDevice.toString()); + if (!_resourceMgr.isGPUDeviceAvailable(host.getId(), groupName.getValue(), requestedVGpuType.getValue())) { + logger.debug("Adding host [{}] to avoid set, because this host does not have required GPU devices available.", + () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); + avoid.addHost(host.getId()); + return true; + } + return false; } - // Reorder hosts in the decreasing order of free capacity. + /** + * Reorder hosts in the decreasing order of free capacity. + */ private List reorderHostsByCapacity(DeploymentPlan plan, List hosts) { Long zoneId = plan.getDataCenterId(); Long clusterId = plan.getClusterId(); - //Get capacity by which we should reorder String capacityTypeToOrder = _configDao.getValue(Config.HostCapacityTypeToOrderClusters.key()); - short capacityType = CapacityVO.CAPACITY_TYPE_CPU; - if("RAM".equalsIgnoreCase(capacityTypeToOrder)){ - capacityType = CapacityVO.CAPACITY_TYPE_MEMORY; - } - List hostIdsByFreeCapacity = _capacityDao.orderHostsByFreeCapacity(zoneId, clusterId, capacityType); - if (logger.isDebugEnabled()) { - logger.debug("List of hosts in descending order of free capacity in the cluster: "+ hostIdsByFreeCapacity); - } + short capacityType = "RAM".equalsIgnoreCase(capacityTypeToOrder) ? CapacityVO.CAPACITY_TYPE_MEMORY : CapacityVO.CAPACITY_TYPE_CPU; - //now filter the given list of Hosts by this ordered list - Map hostMap = new HashMap<>(); - for (Host host : hosts) { - hostMap.put(host.getId(), host); - } - List matchingHostIds = new ArrayList<>(hostMap.keySet()); - - hostIdsByFreeCapacity.retainAll(matchingHostIds); - - List reorderedHosts = new ArrayList<>(); - for(Long id: hostIdsByFreeCapacity){ - reorderedHosts.add(hostMap.get(id)); - } + List hostIdsByFreeCapacity = _capacityDao.orderHostsByFreeCapacity(zoneId, clusterId, capacityType); + logger.debug("List of hosts in descending order of free capacity in the cluster: {}.", hostIdsByFreeCapacity); - return reorderedHosts; + return filterHosts(hosts, hostIdsByFreeCapacity); } private List reorderHostsByNumberOfVms(DeploymentPlan plan, List hosts, Account account) { @@ -406,21 +410,26 @@ private List reorderHostsByNumberOfVms(DeploymentPlan plan, List Long clusterId = plan.getClusterId(); List hostIdsByVmCount = _vmInstanceDao.listHostIdsByVmCount(dcId, podId, clusterId, account.getAccountId()); - if (logger.isDebugEnabled()) { - logger.debug("List of hosts in ascending order of number of VMs: " + hostIdsByVmCount); - } + logger.debug("List of hosts in ascending order of number of VMs: {}.", hostIdsByVmCount); - //now filter the given list of Hosts by this ordered list + return filterHosts(hosts, hostIdsByVmCount); + } + + /** + * Filter the given list of Hosts considering the ordered list + */ + private List filterHosts(List hosts, List orderedHostIdsList) { Map hostMap = new HashMap<>(); + for (Host host : hosts) { hostMap.put(host.getId(), host); } List matchingHostIds = new ArrayList<>(hostMap.keySet()); - hostIdsByVmCount.retainAll(matchingHostIds); + orderedHostIdsList.retainAll(matchingHostIds); List reorderedHosts = new ArrayList<>(); - for (Long id : hostIdsByVmCount) { + for(Long id: orderedHostIdsList){ reorderedHosts.add(hostMap.get(id)); } @@ -434,20 +443,37 @@ public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering off return true; } + /** + * Reorder the host list giving priority to hosts that have the minimum to support the VM's requirements. + */ protected List prioritizeHosts(VMTemplateVO template, ServiceOffering offering, List hosts) { if (template == null) { return hosts; } - // Determine the guest OS category of the template - String templateGuestOSCategory = getTemplateGuestOSCategory(template); + List hostsToCheck = filterHostWithNoHvmIfTemplateRequested(template, hosts); List prioritizedHosts = new ArrayList<>(); - List noHvmHosts = new ArrayList<>(); + List highPriorityHosts = new ArrayList<>(); + List lowPriorityHosts = new ArrayList<>(); + + prioritizeHostsWithMatchingGuestOs(template, hostsToCheck, highPriorityHosts, lowPriorityHosts); + prioritizeHostsByHvmCapability(template, hostsToCheck, prioritizedHosts, highPriorityHosts, lowPriorityHosts); + prioritizeHostsByGpuEnabled(offering, prioritizedHosts); - // If a template requires HVM and a host doesn't support HVM, remove it from consideration + return prioritizedHosts; + } + + /** + * If a template requires HVM and a host doesn't support HVM, remove it from consideration + */ + private List filterHostWithNoHvmIfTemplateRequested(VMTemplateVO template, List hosts) { + List noHvmHosts = new ArrayList<>(); List hostsToCheck = new ArrayList<>(); - if (template.isRequiresHvm()) { + + if (!template.isRequiresHvm()) { + hostsToCheck.addAll(hosts); + } else { for (Host host : hosts) { if (hostSupportsHVM(host)) { hostsToCheck.add(host); @@ -455,65 +481,81 @@ protected List prioritizeHosts(VMTemplateVO template, ServiceOff noHvmHosts.add(host); } } - } else { - hostsToCheck.addAll(hosts); } - if (logger.isDebugEnabled()) { - if (noHvmHosts.size() > 0) { - logger.debug("Not considering hosts: " + noHvmHosts + " to deploy template: " + template + " as they are not HVM enabled"); - } + if (!noHvmHosts.isEmpty()) { + logger.debug("Not considering hosts: " + noHvmHosts + " to deploy template: " + template + " as they are not HVM enabled"); } - // If a host is tagged with the same guest OS category as the template, move it to a high priority list - // If a host is tagged with a different guest OS category than the template, move it to a low priority list - List highPriorityHosts = new ArrayList<>(); - List lowPriorityHosts = new ArrayList<>(); - for (Host host : hostsToCheck) { - String hostGuestOSCategory = getHostGuestOSCategory(host); - if (hostGuestOSCategory == null) { - continue; - } else if (templateGuestOSCategory != null && templateGuestOSCategory.equals(hostGuestOSCategory)) { - highPriorityHosts.add(host); - } else { - lowPriorityHosts.add(host); + + return hostsToCheck; + } + + /** + * If service offering does not request for vGPU, then append all host with GPU to the end of the host priority list. + */ + private void prioritizeHostsByGpuEnabled(ServiceOffering offering, List prioritizedHosts) { + boolean serviceOfferingRequestedVGpu = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) == null; + + if (serviceOfferingRequestedVGpu) { + List gpuEnabledHosts = new ArrayList<>(); + + for (Host host : prioritizedHosts) { + if (_resourceMgr.isHostGpuEnabled(host.getId())) { + gpuEnabledHosts.add(host); + } + } + + if(!gpuEnabledHosts.isEmpty()) { + prioritizedHosts.removeAll(gpuEnabledHosts); + prioritizedHosts.addAll(gpuEnabledHosts); } } + } - hostsToCheck.removeAll(highPriorityHosts); - hostsToCheck.removeAll(lowPriorityHosts); + /** + * Prioritize remaining host by HVM capability. + * + *
    + *
  • If host and template both do not support HVM, put it at the start of the list.
    • + *
  • If the template doesn't require HVM, but the machine supports it, append it to the list.
    • + *
+ */ + private void prioritizeHostsByHvmCapability(VMTemplateVO template, List hostsToCheck, List prioritizedHosts, List highPriorityHosts, List lowPriorityHosts) { - // Prioritize the remaining hosts by HVM capability for (Host host : hostsToCheck) { if (!template.isRequiresHvm() && !hostSupportsHVM(host)) { - // Host and template both do not support hvm, put it as first consideration prioritizedHosts.add(0, host); } else { - // Template doesn't require hvm, but the machine supports it, make it last for consideration prioritizedHosts.add(host); } } - // Merge the lists prioritizedHosts.addAll(0, highPriorityHosts); prioritizedHosts.addAll(lowPriorityHosts); + } - // if service offering is not GPU enabled then move all the GPU enabled hosts to the end of priority list. - if (_serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) == null) { + /** + *
    + *
  • If a host is tagged with the same guest OS category as the template, move it to a high priority list.
    • + *
  • If a host is tagged with a different guest OS category than the template, move it to a low priority list.
    • + *
+ */ + private void prioritizeHostsWithMatchingGuestOs(VMTemplateVO template, List hostsToCheck, List highPriorityHosts, List lowPriorityHosts) { + String templateGuestOSCategory = getTemplateGuestOSCategory(template); + + for (Host host : hostsToCheck) { + String hostGuestOSCategory = getHostGuestOSCategory(host); + + if (StringUtils.equals(templateGuestOSCategory, hostGuestOSCategory)) { + highPriorityHosts.add(host); + } else if (hostGuestOSCategory != null) { + lowPriorityHosts.add(host); - List gpuEnabledHosts = new ArrayList<>(); - // Check for GPU enabled hosts. - for (Host host : prioritizedHosts) { - if (_resourceMgr.isHostGpuEnabled(host.getId())) { - gpuEnabledHosts.add(host); - } - } - // Move GPU enabled hosts to the end of list - if(!gpuEnabledHosts.isEmpty()) { - prioritizedHosts.removeAll(gpuEnabledHosts); - prioritizedHosts.addAll(gpuEnabledHosts); } } - return prioritizedHosts; + + hostsToCheck.removeAll(highPriorityHosts); + hostsToCheck.removeAll(lowPriorityHosts); } protected boolean hostSupportsHVM(Host host) { @@ -582,7 +624,7 @@ public boolean configure(String name, Map params) throws Configu _allocationAlgorithm = allocationAlgorithm; } String value = configs.get("xenserver.check.hvm"); - _checkHvm = value == null ? true : Boolean.parseBoolean(value); + _checkHvm = value == null || Boolean.parseBoolean(value); } return true; } From affcd33708268fdcbd8dd8c1acd372ec3c235c59 Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Mon, 24 Jun 2024 15:01:17 -0300 Subject: [PATCH 2/7] Break into smaller methods random and firfit allocators. --- .../manager/allocator/HostAllocator.java | 33 -- .../admin/host/FindHostsForMigrationCmd.java | 2 +- .../api/command/admin/host/ListHostsCmd.java | 2 +- .../com/cloud/vm/VirtualMachineManager.java | 9 - .../cloud/vm/VirtualMachineManagerImpl.java | 18 -- .../allocator/impl/RandomAllocator.java | 195 ++++++------ .../allocator/impl/FirstFitAllocator.java | 287 +++++++----------- .../allocator/impl/TestingAllocator.java | 21 +- .../cloud/server/ManagementServerImpl.java | 10 +- 9 files changed, 205 insertions(+), 372 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java index 604720aaa290..5d028d31d5b6 100644 --- a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java +++ b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java @@ -22,19 +22,11 @@ import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; -import com.cloud.offering.ServiceOffering; import com.cloud.utils.component.Adapter; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public interface HostAllocator extends Adapter { - /** - * @param UserVm vm - * @param ServiceOffering offering - **/ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - /** * Determines which physical hosts are suitable to * allocate the guest virtual machines on @@ -49,31 +41,6 @@ public interface HostAllocator extends Adapter { public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo); - /** - * Determines which physical hosts are suitable to allocate the guest - * virtual machines on - * - * Allocators must set any other hosts not considered for allocation in the - * ExcludeList avoid. Thus the avoid set and the list of hosts suitable, - * together must cover the entire host set in the cluster. - * - * @param VirtualMachineProfile - * vmProfile - * @param DeploymentPlan - * plan - * @param GuestType - * type - * @param ExcludeList - * avoid - * @param int returnUpTo (use -1 to return all possible hosts) - * @param boolean considerReservedCapacity (default should be true, set to - * false if host capacity calculation should not look at reserved - * capacity) - * @return List List of hosts that are suitable for VM allocation - **/ - - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity); - /** * Determines which physical hosts are suitable to allocate the guest * virtual machines on diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java index db30e4f4c02f..aa855e1f19bf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java @@ -78,7 +78,7 @@ public void execute() { for (Host host : result.first()) { HostForMigrationResponse hostResponse = _responseGenerator.createHostForMigrationResponse(host); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index af87bbf33bb0..f08177302dc7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -212,7 +212,7 @@ protected ListResponse getHostResponses() { for (Host host : result.first()) { HostResponse hostResponse = _responseGenerator.createHostResponse(host, getDetails()); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 04ba9a483e1d..0545b8bf2533 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -157,15 +157,6 @@ void orchestrateStart(String vmUuid, Map pa void advanceReboot(String vmUuid, Map params) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; - /** - * Check to see if a virtual machine can be upgraded to the given service offering - * - * @param vm - * @param offering - * @return true if the host can handle the upgrade, false otherwise - */ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - VirtualMachine findById(long vmId); void storageMigration(String vmUuid, Map volumeToPool); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index d21e8b0fc7b2..a38bdf1e1bd6 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -3519,19 +3519,6 @@ protected void runInContext() { } } - @Override - public boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering) { - boolean isMachineUpgradable = true; - for (final HostAllocator allocator : hostAllocators) { - isMachineUpgradable = allocator.isVirtualMachineUpgradable(vm, offering); - if (!isMachineUpgradable) { - break; - } - } - - return isMachineUpgradable; - } - @Override public void reboot(final String vmUuid, final Map params) throws InsufficientCapacityException, ResourceUnavailableException { try { @@ -3912,11 +3899,6 @@ public void checkIfCanUpgrade(final VirtualMachine vmInstance, final ServiceOffe throw new InvalidParameterValueException("isSystem property is different for current service offering and new service offering"); } - if (!isVirtualMachineUpgradable(vmInstance, newServiceOffering)) { - throw new InvalidParameterValueException("Unable to upgrade virtual machine, not enough resources available " + "for an offering of " + - newServiceOffering.getCpu() + " cpu(s) at " + newServiceOffering.getSpeed() + " Mhz, and " + newServiceOffering.getRamSize() + " MB of memory"); - } - final List currentTags = StringUtils.csvTagsToList(currentDiskOffering.getTags()); final List newTags = StringUtils.csvTagsToList(newDiskOffering.getTags()); if (VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.valueIn(vmInstance.getDataCenterId())) { diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java index 974e0eb726c9..9f6e8dde8d55 100644 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -25,13 +25,10 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.ListUtils; import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.capacity.CapacityManager; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.dao.ClusterDao; import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; @@ -43,7 +40,6 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; @Component @@ -53,142 +49,133 @@ public class RandomAllocator extends AdapterBase implements HostAllocator { @Inject private ResourceManager _resourceMgr; @Inject - private ClusterDao clusterDao; - @Inject - private ClusterDetailsDao clusterDetailsDao; - @Inject private CapacityManager capacityManager; - protected List listHostsByTags(Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { - List taggedHosts = new ArrayList<>(); - if (offeringHostTag != null) { - taggedHosts.addAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag)); - } - if (templateTag != null) { - List templateTaggedHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); - if (taggedHosts.isEmpty()) { - taggedHosts = templateTaggedHosts; - } else { - taggedHosts.retainAll(templateTaggedHosts); - } - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found %d hosts %s with type: %s, zone ID: %d, pod ID: %d, cluster ID: %s, offering host tag(s): %s, template tag: %s", - taggedHosts.size(), - (taggedHosts.isEmpty() ? "" : String.format("(%s)", StringUtils.join(taggedHosts.stream().map(HostVO::getId).toArray(), ","))), - type.name(), dcId, podId, clusterId, offeringHostTag, templateTag)); - } - return taggedHosts; - } - private List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, + protected List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { + if (type == Host.Type.Storage) { + return null; + } + long dcId = plan.getDataCenterId(); Long podId = plan.getPodId(); Long clusterId = plan.getClusterId(); ServiceOffering offering = vmProfile.getServiceOffering(); - List hostsCopy = null; - List suitableHosts = new ArrayList<>(); - if (type == Host.Type.Storage) { - return suitableHosts; - } String offeringHostTag = offering.getHostTag(); - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); - String templateTag = template.getTemplateTag(); - String hostTag = null; - if (ObjectUtils.anyNull(offeringHostTag, templateTag)) { - hostTag = offeringHostTag; - hostTag = hostTag == null ? templateTag : String.format("%s, %s", hostTag, templateTag); - logger.debug(String.format("Looking for hosts in dc [%s], pod [%s], cluster [%s] and complying with host tag(s): [%s]", dcId, podId, clusterId, hostTag)); - } else { - logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId); - } - if (hosts != null) { - // retain all computing hosts, regardless of whether they support routing...it's random after all - hostsCopy = new ArrayList<>(hosts); - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostsCopy.retainAll(listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag)); - } else { - hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId)); - } - } else { - // list all computing hosts, regardless of whether they support routing...it's random after all - if (offeringHostTag != null) { - hostsCopy = listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag); - } else { - hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); - } - } - hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOfferingTags(hostTag)); + VMTemplateVO template = (VMTemplateVO) vmProfile.getTemplate(); + logger.debug("Looking for hosts in zone [{}], pod [{}], cluster [{}].", dcId, podId, clusterId); + + List availableHosts = retrieveHosts(type, (List) hosts, template, offeringHostTag, clusterId, podId, dcId); - if (hostsCopy.isEmpty()) { - logger.info("No suitable host found for VM [{}] in {}.", vmProfile, hostTag); + if (availableHosts.isEmpty()) { + logger.info("No suitable host found for VM [{}] in zone [{}], pod [{}], cluster [{}].", vmProfile, dcId, podId, clusterId); return null; } - logger.debug("Random Allocator found {} hosts", hostsCopy.size()); - if (hostsCopy.isEmpty()) { - return suitableHosts; - } + return filterAvailableHosts(avoid, returnUpTo, considerReservedCapacity, availableHosts, offering); + } - Collections.shuffle(hostsCopy); - for (Host host : hostsCopy) { + protected List filterAvailableHosts(ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity, List availableHosts, ServiceOffering offering) { + logger.debug("Random Allocator found [{}] available hosts. They will be checked if they are in the avoid set and for CPU capability and capacity.", availableHosts::size); + List suitableHosts = new ArrayList<>(); + + Collections.shuffle(availableHosts); + for (Host host : availableHosts) { if (suitableHosts.size() == returnUpTo) { break; } + if (avoid.shouldAvoid(host)) { - if (logger.isDebugEnabled()) { - logger.debug("Host name: " + host.getName() + ", hostId: " + host.getId() + " is in avoid set, skipping this and trying other available hosts"); - } + logger.debug("Host [{}] is in the avoid set, skipping it and trying other available hosts.", () -> host); continue; } - Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - if (!cpuCapabilityAndCapacity.first() || !cpuCapabilityAndCapacity.second()) { - if (logger.isDebugEnabled()) { - logger.debug("Not using host " + host.getId() + "; host has cpu capability? " + cpuCapabilityAndCapacity.first() + ", host has capacity?" + cpuCapabilityAndCapacity.second()); - } + + if (!hostHasCpuCapabilityAndCapacity(considerReservedCapacity, offering, host)) { continue; } - if (logger.isDebugEnabled()) { - logger.debug("Found a suitable host, adding to list: " + host.getId()); - } + + logger.debug("Found the suitable host [{}], adding to list.", () -> host); suitableHosts.add(host); } - if (logger.isDebugEnabled()) { - logger.debug("Random Host Allocator returning " + suitableHosts.size() + " suitable hosts"); - } + + logger.debug("Random Host Allocator returning {} suitable hosts.", suitableHosts::size); return suitableHosts; } + + protected boolean hostHasCpuCapabilityAndCapacity(boolean considerReservedCapacity, ServiceOffering offering, Host host) { + Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); + Boolean hasCpuCapability = cpuCapabilityAndCapacity.first(); + Boolean hasCpuCapacity = cpuCapabilityAndCapacity.second(); + + if (hasCpuCapability && hasCpuCapacity) { + logger.debug("Host {} has enough CPU capability and CPU capacity.", host); + return true; + } + + logger.debug("Not using host [{}]. Does the host have cpu capability? {}. Does the host have capacity? {}.", () -> host, () -> hasCpuCapability, () -> hasCpuCapacity); + return false; + } + + /** + * @return all computing hosts, regardless of whether they support routing. + */ + protected List retrieveHosts(Type type, List hosts, VMTemplateVO template, String offeringHostTag, Long clusterId, Long podId, long dcId) { + List availableHosts; + String templateTag = template.getTemplateTag(); + + if (CollectionUtils.isNotEmpty(hosts)) { + availableHosts = new ArrayList<>(hosts); + } else { + availableHosts = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId); + } + + if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { + retainHostsWithMatchingTags(availableHosts, type, clusterId, podId, dcId, offeringHostTag, templateTag); + } else { + List hostsWithNoRuleTag = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); + logger.debug("Retaining hosts {} because they do not have rule tags.", hostsWithNoRuleTag); + availableHosts.retainAll(hostsWithNoRuleTag); + } + + List hostsWithTagRuleThatMatchComputeOfferingTags = _hostDao.findHostsWithTagRuleThatMatchComputeOfferingTags(offeringHostTag); + logger.debug("Adding hosts {} to the available pool hosts because they match the compute offering's tag.", hostsWithTagRuleThatMatchComputeOfferingTags, offeringHostTag); + availableHosts = ListUtils.union(availableHosts, hostsWithTagRuleThatMatchComputeOfferingTags); + + return availableHosts; + } + + protected void retainHostsWithMatchingTags(List availableHosts, Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { + logger.debug("Hosts {} will be checked for template and host tags compatibility.", availableHosts); + + if (offeringHostTag != null) { + List hostsWithHostTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag); + logger.debug("Retaining hosts {} because they match the offering host tag {}.", hostsWithHostTag, offeringHostTag); + availableHosts.retainAll(hostsWithHostTag); + } + + if (templateTag != null) { + List hostsWithTemplateTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); + logger.debug("Retaining hosts {} because they match the template tag {}.", hostsWithTemplateTag, offeringHostTag); + availableHosts.retainAll(hostsWithTemplateTag); + } + + logger.debug("Remaining hosts after template tag and host tags validations are {}.", availableHosts); + } + @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); } @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { if (CollectionUtils.isEmpty(hosts)) { - if (logger.isDebugEnabled()) { - logger.debug("Random Allocator found 0 hosts as given host list is empty"); - } + logger.debug("Random Allocator found 0 hosts as given host list is empty"); return new ArrayList<>(); } return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity); } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, - Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - return findSuitableHosts(vmProfile, plan, type, avoid, null, returnUpTo, considerReservedCapacity); - } - - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } } diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index c36c26f67b67..9855548d2f07 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -38,8 +38,6 @@ import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; import com.cloud.configuration.Config; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.dao.ClusterDao; import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.gpu.GPU; @@ -62,7 +60,6 @@ import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.vm.UserVmDetailVO; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; @@ -88,10 +85,6 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { @Inject protected ResourceManager _resourceMgr; @Inject - ClusterDao _clusterDao; - @Inject - ClusterDetailsDao _clusterDetailsDao; - @Inject ServiceOfferingDetailsDao _serviceOfferingDetailsDao; @Inject CapacityManager _capacityMgr; @@ -106,11 +99,12 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); } @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, + boolean considerReservedCapacity) { if (type == Host.Type.Storage) { return null; } @@ -119,42 +113,44 @@ public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan pla Long podId = plan.getPodId(); Long clusterId = plan.getClusterId(); ServiceOffering offering = vmProfile.getServiceOffering(); - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); + VMTemplateVO template = (VMTemplateVO) vmProfile.getTemplate(); Account account = vmProfile.getOwner(); String hostTagOnOffering = offering.getHostTag(); String hostTagOnTemplate = template.getTemplateTag(); String paramAsStringToLog = String.format("zone [%s], pod [%s], cluster [%s]", dcId, podId, clusterId); - List suitableHosts = retrieveHosts(vmProfile, type, clusterId, podId, dcId, hostTagOnOffering, hostTagOnTemplate); + List suitableHosts = retrieveHosts(vmProfile, type, (List) hosts, clusterId, podId, dcId, hostTagOnOffering, hostTagOnTemplate); - if (isSuitableHostsEmpty(vmProfile, suitableHosts, paramAsStringToLog)) { + if (suitableHosts.isEmpty()) { + logger.info("No suitable host found for VM [{}] in {}.", vmProfile, paramAsStringToLog); return null; } - addHostsToAvoidSet(type, avoid, clusterId, podId, dcId, suitableHosts); + if (CollectionUtils.isEmpty(hosts)) { + addHostsToAvoidSet(type, avoid, clusterId, podId, dcId, suitableHosts); + } return allocateTo(plan, offering, template, avoid, suitableHosts, returnUpTo, considerReservedCapacity, account); } - private boolean isSuitableHostsEmpty(VirtualMachineProfile vmProfile, List suitableHosts, String paramAsStringToLog) { - if (suitableHosts.isEmpty()) { - logger.info("No suitable host found for VM [{}] in {}.", vmProfile, paramAsStringToLog); - return true; - } - return false; - } - - private List retrieveHosts(VirtualMachineProfile vmProfile, Type type, Long clusterId, Long podId, long dcId, String hostTagOnOffering, String hostTagOnTemplate) { + protected List retrieveHosts(VirtualMachineProfile vmProfile, Type type, List hostsToFilter, Long clusterId, Long podId, long dcId, String hostTagOnOffering, + String hostTagOnTemplate) { String haVmTag = (String) vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); List clusterHosts; + if (CollectionUtils.isNotEmpty(hostsToFilter)) { + clusterHosts = new ArrayList<>(hostsToFilter); + } else { + clusterHosts = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId); + } + if (haVmTag != null) { - clusterHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag); + clusterHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); } else if (ObjectUtils.allNull(hostTagOnOffering, hostTagOnTemplate)) { - clusterHosts = _resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId); + clusterHosts.retainAll(_resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId)); } else { - clusterHosts = retrieveHostsMatchingServiceOfferingAndTemplateTags(hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); + retainHostsMatchingServiceOfferingAndTemplateTags(clusterHosts, hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); } filterHostsWithUefiEnabled(type, vmProfile, clusterId, podId, dcId, clusterHosts); @@ -162,55 +158,43 @@ private List retrieveHosts(VirtualMachineProfile vmProfile, Type type, L addHostsBasedOnTagRules(hostTagOnOffering, clusterHosts); return clusterHosts; + } - private void addHostsBasedOnTagRules(String hostTagOnOffering, List clusterHosts) { + protected void addHostsBasedOnTagRules(String hostTagOnOffering, List clusterHosts) { List hostsWithTagRules = _hostDao.findHostsWithTagRuleThatMatchComputeOfferingTags(hostTagOnOffering); if (CollectionUtils.isEmpty(hostsWithTagRules)) { - logger.info("No hosts found with tag rules matching the compute offering tag {}.", hostTagOnOffering); + logger.info("No hosts found with tag rules matching the compute offering tag [{}].", hostTagOnOffering); + return; } - logger.info("Found hosts {} with tag rules matching the compute offering tag [{}].", hostsWithTagRules, hostTagOnOffering); + logger.info("Found hosts %s with tag rules matching the compute offering tag [{}].", hostsWithTagRules, hostTagOnOffering); clusterHosts.addAll(hostsWithTagRules); } - private List retrieveHostsMatchingServiceOfferingAndTemplateTags(String hostTagOnTemplate, String hostTagOnOffering, Type type, Long clusterId, Long podId, long dcId) { + protected void retainHostsMatchingServiceOfferingAndTemplateTags(List clusterHosts, String hostTagOnTemplate, String hostTagOnOffering, Type type, Long clusterId, + Long podId, long dcId) { boolean hasSvcOfferingTag = hostTagOnOffering != null; boolean hasTemplateTag = hostTagOnTemplate != null; - List clusterHosts; - List hostsMatchingOfferingTag = new ArrayList<>(); - List hostsMatchingTemplateTag = new ArrayList<>(); if (hasSvcOfferingTag) { logger.debug("Looking for hosts having the tag [{}] specified in the Service Offering.", hostTagOnOffering); - hostsMatchingOfferingTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); - logger.debug("Hosts with Service Offering tag [{}] are {}.", hostTagOnOffering, hostsMatchingOfferingTag); + clusterHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering)); + logger.debug("Hosts with Service Offering tag [{}] are {}.", hostTagOnOffering, clusterHosts); } if (hasTemplateTag) { logger.debug("Looking for hosts having the tag [{}] specified in the Template.", hostTagOnTemplate); - hostsMatchingTemplateTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); - logger.debug("Hosts with Template tag [{}] are {}.", hostTagOnTemplate, hostsMatchingTemplateTag); - } - - if (hasSvcOfferingTag && hasTemplateTag) { - hostsMatchingOfferingTag.retainAll(hostsMatchingTemplateTag); - logger.debug("Found {} Hosts satisfying both tags; host IDs are {}.", hostsMatchingOfferingTag.size(), hostsMatchingOfferingTag); - clusterHosts = hostsMatchingOfferingTag; - } else if (hasSvcOfferingTag) { - clusterHosts = hostsMatchingOfferingTag; - } else { - clusterHosts = hostsMatchingTemplateTag; + clusterHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate)); + logger.debug("Hosts with Template tag [{}] are {}.", hostTagOnTemplate, clusterHosts); } - - return clusterHosts; } /** * Add all hosts to the avoid set that were not considered during the allocation */ - private void addHostsToAvoidSet(Type type, ExcludeList avoid, Long clusterId, Long podId, long dcId, List suitableHosts) { + protected void addHostsToAvoidSet(Type type, ExcludeList avoid, Long clusterId, Long podId, long dcId, List suitableHosts) { List allHostsInCluster = _hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); allHostsInCluster.removeAll(suitableHosts); @@ -223,7 +207,7 @@ private void addHostsToAvoidSet(Type type, ExcludeList avoid, Long clusterId, Lo } } - private void filterHostsWithUefiEnabled(Type type, VirtualMachineProfile vmProfile, Long clusterId, Long podId, long dcId, List clusterHosts) { + protected void filterHostsWithUefiEnabled(Type type, VirtualMachineProfile vmProfile, Long clusterId, Long podId, long dcId, List clusterHosts) { UserVmDetailVO userVmDetailVO = _userVmDetailsDao.findDetail(vmProfile.getId(), "UEFI"); if (userVmDetailVO == null) { @@ -242,63 +226,6 @@ private void filterHostsWithUefiEnabled(Type type, VirtualMachineProfile vmProfi clusterHosts.retainAll(hostsMatchingUefiTag); } - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - if (type == Host.Type.Storage) { - return null; - } - - long dcId = plan.getDataCenterId(); - Long podId = plan.getPodId(); - Long clusterId = plan.getClusterId(); - ServiceOffering offering = vmProfile.getServiceOffering(); - VMTemplateVO template = (VMTemplateVO) vmProfile.getTemplate(); - Account account = vmProfile.getOwner(); - - String hostTagOnOffering = offering.getHostTag(); - String hostTagOnTemplate = template.getTemplateTag(); - List suitableHosts = (List) new ArrayList<>(hosts); - - String paramAsStringToLog = String.format("zone [%s], pod [%s], cluster [%s]", dcId, podId, clusterId); - logger.debug("Looking for hosts in {}.", paramAsStringToLog); - - retainHostsMatchingCriteria(vmProfile, type, suitableHosts, clusterId, podId, dcId, hostTagOnOffering, hostTagOnTemplate); - - addHostsBasedOnTagRules(hostTagOnOffering, suitableHosts); - - if (isSuitableHostsEmpty(vmProfile, suitableHosts, paramAsStringToLog)) { - return null; - } - - return allocateTo(plan, offering, template, avoid, suitableHosts, returnUpTo, considerReservedCapacity, account); - } - - private void retainHostsMatchingCriteria(VirtualMachineProfile vmProfile, Type type, List suitableHosts, Long clusterId, Long podId, long dcId, - String hostTagOnOffering, String hostTagOnTemplate) { - String haVmTag = (String) vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); - boolean hasSvcOfferingTag = hostTagOnOffering != null; - boolean hasTemplateTag = hostTagOnTemplate != null; - - if (haVmTag != null) { - suitableHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); - } else if (ObjectUtils.allNull(hostTagOnOffering, hostTagOnTemplate)) { - suitableHosts.retainAll(_resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId)); - } else { - if (hasSvcOfferingTag) { - logger.debug("Looking for hosts having the tag [{}] specified in the Service Offering.", hostTagOnOffering); - suitableHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering)); - logger.debug("Hosts with Service Offering tag [{}] are {}.", hostTagOnOffering, suitableHosts); - } - - if (hasTemplateTag) { - logger.debug("Looking for hosts having the tag [{}] specified in the Template.", hostTagOnTemplate); - suitableHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate)); - logger.debug("Hosts with Template tag [{}] are {}.", hostTagOnTemplate, suitableHosts); - } - } - } - protected List allocateTo(DeploymentPlan plan, ServiceOffering offering, VMTemplateVO template, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity, Account account) { switch (_allocationAlgorithm) { @@ -315,22 +242,20 @@ protected List allocateTo(DeploymentPlan plan, ServiceOffering offering, V break; } - logger.debug("FirstFitAllocator has {} hosts to check for allocation: {}.", hosts.size(), hosts); - + logger.debug("FirstFitAllocator has {} hosts to check for allocation {}.", hosts.size(), hosts); hosts = prioritizeHosts(template, offering, hosts); - logger.debug("Found {} hosts for allocation after prioritization: {}.", hosts.size(), hosts); - logger.debug("Looking for frequency {} MHz and RAM {} MB.", () -> offering.getCpu() * offering.getSpeed(), offering::getRamSize); List suitableHosts = checkHostsCompatibilities(offering, avoid, hosts, returnUpTo, considerReservedCapacity); - - logger.debug("Host Allocator returning {} suitable hosts", suitableHosts.size()); + logger.debug("Host Allocator returning {} suitable hosts.", suitableHosts.size()); return suitableHosts; } - private List checkHostsCompatibilities(ServiceOffering offering, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { + + protected List checkHostsCompatibilities(ServiceOffering offering, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { List suitableHosts = new ArrayList<>(); + logger.debug("Checking compatibility for the following hosts {}.", suitableHosts); for (Host host : hosts) { if (suitableHosts.size() == returnUpTo) { @@ -338,14 +263,14 @@ private List checkHostsCompatibilities(ServiceOffering offering, ExcludeLi } if (avoid.shouldAvoid(host)) { - logger.debug("Host [{}] is in avoid set, skipping this and trying other available hosts", - () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); + logger.debug("Host [{}] is in avoid set, skipping this and trying other available hosts.", () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, + "uuid", "name")); continue; } if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)) { logger.debug("Adding host [{}] to the avoid set because this host already has the max number of running (user and/or system) VMs.", - () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); + () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); avoid.addHost(host.getId()); continue; } @@ -354,21 +279,32 @@ private List checkHostsCompatibilities(ServiceOffering offering, ExcludeLi continue; } - Pair cpuCapabilityAndCapacity = _capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - if (cpuCapabilityAndCapacity.first() && cpuCapabilityAndCapacity.second()) { - logger.debug("Found a suitable host, adding to list host [{}].", () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); - suitableHosts.add(host); - } else { - logger.debug("Not using host {}; host has cpu capability? {}, host has capacity? {}.", - () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name"), cpuCapabilityAndCapacity::first, cpuCapabilityAndCapacity::second); - avoid.addHost(host.getId()); - } + addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(offering, avoid, considerReservedCapacity, host, suitableHosts); } return suitableHosts; } - private boolean offeringRequestedVGpuAndHostDoesNotHaveIt(ServiceOffering offering, ExcludeList avoid, Host host) { + /** + * Adds hosts with CPU capability and CPU capacity to the suitable hosts list. Otherwise, the host is added to the avoid list. + */ + protected void addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(ServiceOffering offering, ExcludeList avoid, boolean considerReservedCapacity, Host host, List suitableHosts) { + logger.debug("Looking for CPU frequency {} MHz and RAM {} MB.", () -> offering.getCpu() * offering.getSpeed(), () -> offering.getRamSize()); + Pair cpuCapabilityAndCapacity = _capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); + Boolean hasCpuCapability = cpuCapabilityAndCapacity.first(); + Boolean hasCpuCapacity = cpuCapabilityAndCapacity.second(); + + if (hasCpuCapability && hasCpuCapacity) { + logger.debug("Found a suitable host, adding to list host [{}].", () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); + suitableHosts.add(host); + } else { + logger.debug("Not using host {}; host has cpu capability? {}, host has capacity? {}.", () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", + "name"), () -> hasCpuCapability, () -> hasCpuCapacity); + avoid.addHost(host.getId()); + } + } + + protected boolean offeringRequestedVGpuAndHostDoesNotHaveIt(ServiceOffering offering, ExcludeList avoid, Host host) { long serviceOfferingId = offering.getId(); ServiceOfferingDetailsVO requestedVGpuType = _serviceOfferingDetailsDao.findDetail(serviceOfferingId, GPU.Keys.vgpuType.toString()); @@ -377,7 +313,7 @@ private boolean offeringRequestedVGpuAndHostDoesNotHaveIt(ServiceOffering offeri } ServiceOfferingDetailsVO groupName = _serviceOfferingDetailsDao.findDetail(serviceOfferingId, GPU.Keys.pciDevice.toString()); - if (!_resourceMgr.isGPUDeviceAvailable(host.getId(), groupName.getValue(), requestedVGpuType.getValue())) { + if(!_resourceMgr.isGPUDeviceAvailable(host.getId(), groupName.getValue(), requestedVGpuType.getValue())){ logger.debug("Adding host [{}] to avoid set, because this host does not have required GPU devices available.", () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); avoid.addHost(host.getId()); @@ -425,7 +361,6 @@ private List filterHosts(List hosts, List hostMap.put(host.getId(), host); } List matchingHostIds = new ArrayList<>(hostMap.keySet()); - orderedHostIdsList.retainAll(matchingHostIds); List reorderedHosts = new ArrayList<>(); @@ -436,13 +371,6 @@ private List filterHosts(List hosts, List return reorderedHosts; } - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } - /** * Reorder the host list giving priority to hosts that have the minimum to support the VM's requirements. */ @@ -458,58 +386,72 @@ protected List prioritizeHosts(VMTemplateVO template, ServiceOff List lowPriorityHosts = new ArrayList<>(); prioritizeHostsWithMatchingGuestOs(template, hostsToCheck, highPriorityHosts, lowPriorityHosts); - prioritizeHostsByHvmCapability(template, hostsToCheck, prioritizedHosts, highPriorityHosts, lowPriorityHosts); + hostsToCheck.removeAll(highPriorityHosts); + hostsToCheck.removeAll(lowPriorityHosts); + + prioritizeHostsByHvmCapability(template, hostsToCheck, prioritizedHosts); + prioritizedHosts.addAll(0, highPriorityHosts); + prioritizedHosts.addAll(lowPriorityHosts); + prioritizeHostsByGpuEnabled(offering, prioritizedHosts); return prioritizedHosts; } + /** - * If a template requires HVM and a host doesn't support HVM, remove it from consideration + * If a template requires HVM and a host doesn't support HVM, remove it from consideration. */ - private List filterHostWithNoHvmIfTemplateRequested(VMTemplateVO template, List hosts) { - List noHvmHosts = new ArrayList<>(); + protected List filterHostWithNoHvmIfTemplateRequested(VMTemplateVO template, List hosts) { List hostsToCheck = new ArrayList<>(); if (!template.isRequiresHvm()) { + logger.debug("Template [{}] does not require HVM, therefore, the hosts {} will not be checked for HVM compatibility.", template, hostsToCheck); hostsToCheck.addAll(hosts); - } else { - for (Host host : hosts) { - if (hostSupportsHVM(host)) { - hostsToCheck.add(host); - } else { - noHvmHosts.add(host); - } + return hostsToCheck; + } + + List noHvmHosts = new ArrayList<>(); + logger.debug("Template [{}] requires HVM, therefore, the hosts %s will be checked for HVM compatibility.", template, hostsToCheck); + + for (Host host : hosts) { + if (hostSupportsHVM(host)) { + hostsToCheck.add(host); + } else { + noHvmHosts.add(host); } } if (!noHvmHosts.isEmpty()) { - logger.debug("Not considering hosts: " + noHvmHosts + " to deploy template: " + template + " as they are not HVM enabled"); + logger.debug("Not considering hosts {} to deploy VM using template {} as they are not HVM enabled.", noHvmHosts, template); } return hostsToCheck; } + /** - * If service offering does not request for vGPU, then append all host with GPU to the end of the host priority list. + * If service offering did not request for vGPU, then move all host with GPU to the end of the host priority list. */ - private void prioritizeHostsByGpuEnabled(ServiceOffering offering, List prioritizedHosts) { - boolean serviceOfferingRequestedVGpu = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) == null; + protected void prioritizeHostsByGpuEnabled(ServiceOffering offering, List prioritizedHosts) { + boolean serviceOfferingRequestedVGpu = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) != null; if (serviceOfferingRequestedVGpu) { - List gpuEnabledHosts = new ArrayList<>(); + return; + } - for (Host host : prioritizedHosts) { - if (_resourceMgr.isHostGpuEnabled(host.getId())) { - gpuEnabledHosts.add(host); - } - } + List gpuEnabledHosts = new ArrayList<>(); - if(!gpuEnabledHosts.isEmpty()) { - prioritizedHosts.removeAll(gpuEnabledHosts); - prioritizedHosts.addAll(gpuEnabledHosts); + for (Host host : prioritizedHosts) { + if (_resourceMgr.isHostGpuEnabled(host.getId())) { + gpuEnabledHosts.add(host); } } + + if (!gpuEnabledHosts.isEmpty()) { + prioritizedHosts.removeAll(gpuEnabledHosts); + prioritizedHosts.addAll(gpuEnabledHosts); + } } /** @@ -520,8 +462,7 @@ private void prioritizeHostsByGpuEnabled(ServiceOffering offering, List pr *
  • If the template doesn't require HVM, but the machine supports it, append it to the list.
    • * */ - private void prioritizeHostsByHvmCapability(VMTemplateVO template, List hostsToCheck, List prioritizedHosts, List highPriorityHosts, List lowPriorityHosts) { - + protected void prioritizeHostsByHvmCapability(VMTemplateVO template, List hostsToCheck, List prioritizedHosts) { for (Host host : hostsToCheck) { if (!template.isRequiresHvm() && !hostSupportsHVM(host)) { prioritizedHosts.add(0, host); @@ -529,18 +470,16 @@ private void prioritizeHostsByHvmCapability(VMTemplateVO template, List ho prioritizedHosts.add(host); } } - - prioritizedHosts.addAll(0, highPriorityHosts); - prioritizedHosts.addAll(lowPriorityHosts); } + /** *
      *
    • If a host is tagged with the same guest OS category as the template, move it to a high priority list.
      • *
    • If a host is tagged with a different guest OS category than the template, move it to a low priority list.
      • *
    */ - private void prioritizeHostsWithMatchingGuestOs(VMTemplateVO template, List hostsToCheck, List highPriorityHosts, List lowPriorityHosts) { + protected void prioritizeHostsWithMatchingGuestOs(VMTemplateVO template, List hostsToCheck, List highPriorityHosts, List lowPriorityHosts) { String templateGuestOSCategory = getTemplateGuestOSCategory(template); for (Host host : hostsToCheck) { @@ -550,14 +489,11 @@ private void prioritizeHostsWithMatchingGuestOs(VMTemplateVO template, List params) throws Configu } return true; } - - @Override - public boolean start() { - return true; - } - - @Override - public boolean stop() { - return true; - } - } diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java index fd8e65e24761..04a0b182b7b1 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java @@ -28,32 +28,24 @@ import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.host.dao.HostDao; -import com.cloud.offering.ServiceOffering; import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public class TestingAllocator extends AdapterBase implements HostAllocator { @Inject HostDao _hostDao; - Long _computingHost; Long _storageHost; Long _routingHost; @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); } @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, considerReservedCapacity); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - List availableHosts = new ArrayList(); + boolean considerReservedCapacity) { + List availableHosts = new ArrayList<>(); Host host = null; if (type == Host.Type.Routing && _routingHost != null) { host = _hostDao.findById(_routingHost); @@ -66,13 +58,6 @@ public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan pla return availableHosts; } - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } - @Override public boolean configure(String name, Map params) { String value = (String)params.get(Host.Type.Routing.toString()); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index a485cfe0c66c..d14b7a3844c8 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -1590,11 +1590,7 @@ public Ternary, Integer>, List, Map, Integer>, List, Map(otherHosts, suitableHosts, requiresStorageMotion); From 82ce32f4384d63508c0fc71bddad83ad5f4f247d Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Mon, 24 Jun 2024 15:01:40 -0300 Subject: [PATCH 3/7] Added unit tests for random and firstfit allocators --- .../allocator/impl/RandomAllocatorTest.java | 333 ++++++++- .../allocator/impl/FirstFitAllocatorTest.java | 638 ++++++++++++++++++ 2 files changed, 958 insertions(+), 13 deletions(-) create mode 100644 server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java diff --git a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java index 538d7157184a..a16a6391d8d3 100644 --- a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java +++ b/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java @@ -17,8 +17,19 @@ package com.cloud.agent.manager.allocator.impl; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import com.cloud.agent.manager.allocator.HostAllocator; +import com.cloud.capacity.CapacityManager; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.utils.Pair; +import com.cloud.vm.VirtualMachineProfile; import org.apache.commons.collections.CollectionUtils; import org.junit.Assert; import org.junit.Test; @@ -26,6 +37,7 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import com.cloud.host.Host; @@ -37,9 +49,50 @@ public class RandomAllocatorTest { @Mock HostDao hostDao; + + @Spy @InjectMocks RandomAllocator randomAllocator; + @Mock + ResourceManager resourceManagerMock; + + @Mock + CapacityManager capacityManagerMock; + + private final Host.Type type = Host.Type.Routing; + + private final Long clusterId = 1L; + + private final Long podId = 1L; + + private final Long zoneId = 1L; + + private final List emptyList = new ArrayList<>(); + + private final String hostTag = "hostTag"; + + private final String templateTag = "templateTag"; + + private final HostVO host1 = Mockito.mock(HostVO.class); + + private final HostVO host2 = Mockito.mock(HostVO.class); + + private final HostVO host3 = Mockito.mock(HostVO.class); + + private final VMTemplateVO vmTemplateVO = Mockito.mock(VMTemplateVO.class); + + private final ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + + private final DeploymentPlanner.ExcludeList excludeList = Mockito.mock(DeploymentPlanner.ExcludeList.class); + + private final VirtualMachineProfile virtualMachineProfile = Mockito.mock(VirtualMachineProfile.class); + + private final DeploymentPlan deploymentPlan = Mockito.mock(DeploymentPlan.class); + + private final boolean considerReservedCapacity = true; + + @Test public void testListHostsByTags() { Host.Type type = Host.Type.Routing; @@ -51,30 +104,284 @@ public void testListHostsByTags() { Mockito.when(hostDao.listByHostTag(type, id, id, id, offeringTag)).thenReturn(List.of(host1, host2)); // No template tagged host + ArrayList noTemplateTaggedHosts = new ArrayList<>(Arrays.asList(host1, host2)); Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(new ArrayList<>()); - List result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); + randomAllocator.retainHostsWithMatchingTags(noTemplateTaggedHosts, type, id, id, id, offeringTag, templateTag); + Assert.assertTrue(CollectionUtils.isEmpty(noTemplateTaggedHosts)); // Different template tagged host + ArrayList differentTemplateTaggedHost = new ArrayList<>(Arrays.asList(host1, host2)); HostVO host3 = Mockito.mock(HostVO.class); Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host3)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); + randomAllocator.retainHostsWithMatchingTags(differentTemplateTaggedHost, type, id, id, id, offeringTag, templateTag); + Assert.assertTrue(CollectionUtils.isEmpty(differentTemplateTaggedHost)); // Matching template tagged host + ArrayList matchingTemplateTaggedHost = new ArrayList<>(Arrays.asList(host1, host2)); Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host1)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); + randomAllocator.retainHostsWithMatchingTags(matchingTemplateTaggedHost, type, id, id, id, offeringTag, templateTag); + Assert.assertFalse(CollectionUtils.isEmpty(matchingTemplateTaggedHost)); + Assert.assertEquals(1, matchingTemplateTaggedHost.size()); // No template tag - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, null); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(2, result.size()); + ArrayList noTemplateTag = new ArrayList<>(Arrays.asList(host1, host2)); + randomAllocator.retainHostsWithMatchingTags(noTemplateTag, type, id, id, id, offeringTag, null); + Assert.assertFalse(CollectionUtils.isEmpty(noTemplateTag)); + Assert.assertEquals(2, noTemplateTag.size()); // No offering tag - result = randomAllocator.listHostsByTags(type, id, id, id, null, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); + ArrayList noOfferingTag = new ArrayList<>(Arrays.asList(host1, host2)); + randomAllocator.retainHostsWithMatchingTags(noOfferingTag, type, id, id, id, null, templateTag); + Assert.assertFalse(CollectionUtils.isEmpty(noOfferingTag)); + Assert.assertEquals(1, noOfferingTag.size()); + } + + @Test + public void findSuitableHostsTestHostTypeStorageShouldReturnNull() { + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, Host.Type.Storage, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void findSuitableHostsTestNoAvailableHostsShouldReturnNull() { + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(emptyList).when(randomAllocator).retrieveHosts(Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.any(VMTemplateVO.class), Mockito.anyString(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void findSuitableHostsTestAvailableHostsShouldCallFilterAvailableHostsOnce() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(hosts).when(randomAllocator).retrieveHosts(Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.any(VMTemplateVO.class), Mockito.anyString(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hosts).when(randomAllocator).filterAvailableHosts(Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyList(), Mockito.any(ServiceOffering.class)); + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(randomAllocator, Mockito.times(1)).filterAvailableHosts(Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyList(), Mockito.any(ServiceOffering.class)); + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestAvailableHostsReachedReturnUpToLimitShouldReturnOnlyHostsWithinLimit() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = 1; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestReturnUpToAllShouldReturnAllAvailableHosts() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestHost1InAvoidShouldOnlyReturnHost2() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(true).when(excludeList).shouldAvoid(host1); + Mockito.doReturn(false).when(excludeList).shouldAvoid(host2); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + } + + @Test + public void filterAvailableHostsTestOnlyHost2HasCpuCapacityAndCapabilityShouldReturnOnlyHost2() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(false).when(randomAllocator).hostHasCpuCapabilityAndCapacity(considerReservedCapacity, serviceOffering, host1); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(considerReservedCapacity, serviceOffering, host2); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityAndCpuCapacityShouldReturnTrue() { + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = randomAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertTrue(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityButNoCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = randomAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityButHasCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = randomAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityAndCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = randomAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + @Test + public void retrieveHostsTestProvidedHostsNullAndNoHostTagAndNoTagRuleShouldOnlyReturnHostsWithNoTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullAndOnlyHostTagsRulesShouldReturnHostsThatMatchRuleTagsAndHostsWithNoTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host2)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host2, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullProvidedHostTagsNotNullAndNoHostWithMatchingRuleTagsShouldReturnHostWithMatchingTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullProvidedHostTagsNotNullAndHostWithMatchingRuleTagsShouldReturnHostWithHostMatchingTagsAndRuleTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host3, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullAndNoHostTagAndNoTagRuleShouldOnlyReturnHostsWithNoTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullAndOnlyHostTagsRulesShouldReturnHostsThatMatchRuleTagsAndHostsWithNoTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host2)); + + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host2, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullProvidedHostTagsNotNullAndNoHostWithMatchingRuleTagsShouldReturnHostWithMatchingTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullNotProvidedHostTagsNotNullAndHostWithMatchingRuleTagsShouldReturnHostWithHostMatchingTagsAndRuleTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host3, availableHosts.get(1)); } } diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java new file mode 100644 index 000000000000..3921fa028bef --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java @@ -0,0 +1,638 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import com.cloud.agent.manager.allocator.HostAllocator; +import com.cloud.capacity.CapacityManager; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingDetailsVO; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.vm.UserVmDetailVO; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.UserVmDetailsDao; +import org.apache.cloudstack.api.ApiConstants; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@RunWith(MockitoJUnitRunner.class) +public class FirstFitAllocatorTest { + + @Mock + HostDao hostDaoMock; + + @Mock + ResourceManager resourceManagerMock; + + @Mock + UserVmDetailsDao userVmDetailsDaoMock; + + @Mock + CapacityManager capacityManagerMock; + + @Mock + ServiceOfferingDetailsDao serviceOfferingDetailsDao; + + @Spy + @InjectMocks + FirstFitAllocator firstFitAllocatorSpy; + + private final Host.Type type = Host.Type.Routing; + + private final Long clusterId = 1L; + + private final Long podId = 1L; + + private final Long dcId = 1L; + + private final List emptyList = new ArrayList<>(); + + private final String hostTag = "hostTag"; + + private final String templateTag = "templateTag"; + + private final HostVO host1 = Mockito.mock(HostVO.class); + + private final HostVO host2 = Mockito.mock(HostVO.class); + + private final HostVO host3 = Mockito.mock(HostVO.class); + + private final ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + + private final DeploymentPlanner.ExcludeList excludeList = Mockito.mock(DeploymentPlanner.ExcludeList.class); + + private final VirtualMachineProfile virtualMachineProfile = Mockito.mock(VirtualMachineProfile.class); + + private final VMTemplateVO vmTemplateVO = Mockito.mock(VMTemplateVO.class); + + private final Account account = Mockito.mock(Account.class); + + private final DeploymentPlan deploymentPlan = Mockito.mock(DeploymentPlan.class); + + private final boolean considerReservedCapacity = true; + + @Test + public void allocateToTestHostTypeStorageShouldReturnNull() { + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, Host.Type.Storage, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void allocateToTestSuitableHostsEmptyShouldReturnNull() { + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(emptyList).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void allocateToTestSuitableHostsNotEmptyShouldCallAllocateToMethod() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).allocateTo(Mockito.any(DeploymentPlan.class), Mockito.any(ServiceOffering.class), Mockito.any(VMTemplateVO.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyList(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.any(Account.class)); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).allocateTo(deploymentPlan, serviceOffering, vmTemplateVO, excludeList, hosts, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity, account); + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void allocateToTestProvidedHostsNotNullShouldCallAddHostsToAvoidSetMethod() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).allocateTo(Mockito.any(DeploymentPlan.class), Mockito.any(ServiceOffering.class), Mockito.any(VMTemplateVO.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyList(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.any(Account.class)); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndHaTagNotNullShouldReturnOnlyHostsWithHaTag() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithHaTag = new ArrayList<>(Arrays.asList(host1, host2)); + String hostVmTag = "haVmTag"; + + Mockito.doReturn(hostVmTag).when(virtualMachineProfile).getParameter(Mockito.any(VirtualMachineProfile.Param.class)); + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithHaTag).when(hostDaoMock).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, hostTag, templateTag); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNotNullAndHaTagNotNullShouldReturnOnlyHostsToFilterWithHaTag() { + List hostsToFilter = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithHaTag = new ArrayList<>(Arrays.asList(host1, host2)); + String hostVmTag = "haVmTag"; + + Mockito.doReturn(hostVmTag).when(virtualMachineProfile).getParameter(Mockito.any(VirtualMachineProfile.Param.class)); + Mockito.doReturn(hostsWithHaTag).when(hostDaoMock).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, hostsToFilter, clusterId, podId, dcId, hostTag, templateTag); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndNoHaTagAndNoHostTagShouldReturnOnlyAllUpAndEnabledNonHaHosts() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List upAndEnabledHostsWithNoHa = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(upAndEnabledHostsWithNoHa).when(resourceManagerMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.nullable(String.class), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, null, null); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndNoHaTagWithHostTagShouldCallRetainHostsMatchingServiceOfferingAndTemplateTags() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doNothing().when(firstFitAllocatorSpy).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.anyString(), Mockito.anyString(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, hostTag, templateTag); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.anyString(), Mockito.anyString(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + } + + @Test + public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsEmptyShouldNotAddToSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(emptyList).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); + firstFitAllocatorSpy.addHostsBasedOnTagRules(hostTag, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + } + + @Test + public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsNotEmptyShouldAddToSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsMatchingRuleTag = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(hostsMatchingRuleTag).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); + firstFitAllocatorSpy.addHostsBasedOnTagRules(hostTag, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagShouldRetainHostsWithServiceOfferingTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingTags = new ArrayList<>(Arrays.asList(host1, host3)); + String hostTagOnTemplate = "hostTagOnTemplate"; + String hostTagOnOffering = null; + + Mockito.doReturn(hostsWithMathingTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + firstFitAllocatorSpy.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagAndHasHostTagOnTemplateShouldRetainHostsWithServiceOfferingTagAndTemplateTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); + List hostsWithMathingTemplateTags = new ArrayList<>(Arrays.asList(host1, host2)); + String hostTagOnTemplate = "hostTagOnTemplate"; + String hostTagOnOffering = "hostTagOnOffering"; + + Mockito.doReturn(hostsWithMathingTemplateTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); + firstFitAllocatorSpy.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasHostTagOnTemplateShouldRetainHostsWithTemplateTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); + String hostTagOnTemplate = null; + String hostTagOnOffering = "hostTagOnOffering"; + + Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); + firstFitAllocatorSpy.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestNoServiceTagAndNoTemplateTagShouldHaveAllSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + String hostTagOnTemplate = null; + String hostTagOnOffering = null; + + firstFitAllocatorSpy.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void addHostsToAvoidSetTestAllHostsWereConsideredForAllocationShouldNotAddAnyHostToTheAvoidSet() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(suitableHosts).when(hostDaoMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.nullable(String.class)); + firstFitAllocatorSpy.addHostsToAvoidSet(type, excludeList, clusterId, podId, dcId, suitableHosts); + + Assert.assertTrue(excludeList.getHostsToAvoid().isEmpty()); + } + + @Test + public void addHostsToAvoidSetTestNotAllHostsWereConsideredForAllocationShouldAddHostToTheAvoidSet() { + List allUpAndEnabledNonHAHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List consideredHosts = new ArrayList<>(Arrays.asList(host2, host3)); + + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); + Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); + Mockito.doReturn(allUpAndEnabledNonHAHosts).when(hostDaoMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.nullable(String.class)); + firstFitAllocatorSpy.addHostsToAvoidSet(type, excludeList, clusterId, podId, dcId, consideredHosts); + + Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); + Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); + } + + @Test + public void filterHostsWithUefiEnabledTestNoDetailWithUefiShouldNotFilterAnyHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + UserVmDetailVO userVmDetailVO = null; + + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithInvalidModeShouldNotFilterAnyHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + UserVmDetailVO userVmDetailVO = Mockito.mock(UserVmDetailVO.class); + String bootMode = "Invalid mode"; + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithLegacyModeShouldFilterHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List uefiHosts = new ArrayList<>(Arrays.asList(host2, host3)); + UserVmDetailVO userVmDetailVO = Mockito.mock(UserVmDetailVO.class); + String bootMode = ApiConstants.BootMode.LEGACY.toString(); + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(uefiHosts).when(hostDaoMock).listByHostCapability(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithSecureModeShouldFilterHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List uefiHosts = new ArrayList<>(Arrays.asList(host2, host3)); + UserVmDetailVO userVmDetailVO = Mockito.mock(UserVmDetailVO.class); + String bootMode = ApiConstants.BootMode.SECURE.toString(); + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(uefiHosts).when(hostDaoMock).listByHostCapability(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void addHostToSuitableHostIfHasCpuCapacityAndCpuCapabilityTestHostHasCapacityAndCapabilityShouldBeAddedToSuitableHosts() { + List suitableHosts = new ArrayList<>(); + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + firstFitAllocatorSpy.addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(serviceOffering, excludeList, considerReservedCapacity, host1, suitableHosts); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + } + + @Test + public void addHostToSuitableHostIfHasCpuCapacityAndCpuCapabilityTestHostDoesNotHaveCapacityAndCapabilityShouldBeAddedToTheAvoidList() { + List suitableHosts = new ArrayList<>(); + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); + Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + firstFitAllocatorSpy.addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(serviceOffering, excludeList, considerReservedCapacity, host1, suitableHosts); + + Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); + Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); + } + + @Test + public void addHostToSuitableHostIfHasCpuCapacityAndCpuCapabilityTestHostHasCapacityButNoCapabilityShouldBeAddedToTheAvoidList() { + List suitableHosts = new ArrayList<>(); + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); + Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + firstFitAllocatorSpy.addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(serviceOffering, excludeList, considerReservedCapacity, host1, suitableHosts); + + Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); + Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); + } + + @Test + public void addHostToSuitableHostIfHasCpuCapacityAndCpuCapabilityTestHostDoesNotHaveCapacityNoCapabilityShouldBeAddedToTheAvoidList() { + List suitableHosts = new ArrayList<>(); + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); + Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + firstFitAllocatorSpy.addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(serviceOffering, excludeList, considerReservedCapacity, host1, suitableHosts); + + Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); + Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); + } + + @Test + public void offeringRequestedVGpuAndHostDoesNotHaveItTestNoVGpuRequestedShouldReturnFalse() { + ServiceOfferingDetailsVO requestedVGpuType = null; + + Mockito.doReturn(1L).when(serviceOffering).getId(); + Mockito.doReturn(requestedVGpuType).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + boolean result = firstFitAllocatorSpy.offeringRequestedVGpuAndHostDoesNotHaveIt(serviceOffering, excludeList, host1); + + Assert.assertFalse(result); + } + + @Test + public void offeringRequestedVGpuAndHostDoesNotHaveItTestVGpuRequestedButHostDoesNotHaveItShouldAddTheHostToTheAvoidListAndReturnTrue() { + ServiceOfferingDetailsVO requestedVGpuType = Mockito.mock(ServiceOfferingDetailsVO.class); + + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); + Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); + Mockito.doReturn(1L).when(serviceOffering).getId(); + Mockito.doReturn(requestedVGpuType).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(false).when(resourceManagerMock).isGPUDeviceAvailable(Mockito.anyLong(), Mockito.nullable(String.class), Mockito.nullable(String.class)); + boolean result = firstFitAllocatorSpy.offeringRequestedVGpuAndHostDoesNotHaveIt(serviceOffering, excludeList, host1); + + Assert.assertTrue(result); + Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); + Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); + } + + @Test + public void offeringRequestedVGpuAndHostDoesNotHaveItTestVGpuRequestedAndHostDoesHaveItShouldReturnFalse() { + ServiceOfferingDetailsVO requestedVGpuType = Mockito.mock(ServiceOfferingDetailsVO.class); + + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doReturn(1L).when(serviceOffering).getId(); + Mockito.doReturn(requestedVGpuType).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(true).when(resourceManagerMock).isGPUDeviceAvailable(Mockito.anyLong(), Mockito.nullable(String.class), Mockito.nullable(String.class)); + boolean result = firstFitAllocatorSpy.offeringRequestedVGpuAndHostDoesNotHaveIt(serviceOffering, excludeList, host1); + + Assert.assertFalse(result); + Mockito.verify(excludeList, Mockito.never()).addHost(Mockito.anyLong()); + } + + @Test + public void filterHostWithNoHvmIfTemplateRequestedTestTemplateDoesNotRequireHvm() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(false).when(vmTemplateVO).isRequiresHvm(); + List suitableHosts = firstFitAllocatorSpy.filterHostWithNoHvmIfTemplateRequested(vmTemplateVO, hosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostWithNoHvmIfTemplateRequestedTestTemplateRequiresHvmShouldReturnOnlyHvmHosts() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(true).when(vmTemplateVO).isRequiresHvm(); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host1); + Mockito.doReturn(false).when(firstFitAllocatorSpy).hostSupportsHVM(host2); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host3); + List suitableHosts = firstFitAllocatorSpy.filterHostWithNoHvmIfTemplateRequested(vmTemplateVO, hosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingRequestedVGpuShouldDoNothing() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + ServiceOfferingDetailsVO requestedVGpuType = Mockito.mock(ServiceOfferingDetailsVO.class); + + Mockito.doReturn(requestedVGpuType).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, hosts); + + Assert.assertEquals(3, hosts.size()); + Assert.assertEquals(host1, hosts.get(0)); + Assert.assertEquals(host2, hosts.get(1)); + Assert.assertEquals(host3, hosts.get(2)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingDidNotRequestVGpuShouldReorderList() { + List allHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + ServiceOfferingDetailsVO requestedVGpuType = null; + + Mockito.doReturn(requestedVGpuType).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doReturn(2L).when(host2).getId(); + Mockito.doReturn(3L).when(host3).getId(); + Mockito.doReturn(true).when(resourceManagerMock).isHostGpuEnabled(1L); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(2L); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(3L); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, allHosts); + + Assert.assertEquals(3, allHosts.size()); + Assert.assertEquals(host2, allHosts.get(0)); + Assert.assertEquals(host3, allHosts.get(1)); + Assert.assertEquals(host1, allHosts.get(2)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingDidNotRequestVGpuShouldNotReorderListIfThereIsNoHostWithVGpu() { + List allHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + ServiceOfferingDetailsVO requestedVGpuType = null; + + Mockito.doReturn(requestedVGpuType).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doReturn(2L).when(host2).getId(); + Mockito.doReturn(3L).when(host3).getId(); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(Mockito.anyLong()); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, allHosts); + + Assert.assertEquals(3, allHosts.size()); + Assert.assertEquals(host1, allHosts.get(0)); + Assert.assertEquals(host2, allHosts.get(1)); + Assert.assertEquals(host3, allHosts.get(2)); + } + + @Test + public void prioritizeHostsByHvmCapabilityTestTemplateDidNotRequestedHvmShouldPutHostThatDoesNotSupportHvmInStartOfThePriorityList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List prioritizedHosts = new ArrayList<>(); + + Mockito.doReturn(false).when(vmTemplateVO).isRequiresHvm(); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host1); + Mockito.doReturn(false).when(firstFitAllocatorSpy).hostSupportsHVM(host2); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host3); + firstFitAllocatorSpy.prioritizeHostsByHvmCapability(vmTemplateVO, hostsToCheck, prioritizedHosts); + + Assert.assertEquals(3, prioritizedHosts.size()); + Assert.assertEquals(host2, prioritizedHosts.get(0)); + Assert.assertEquals(host1, prioritizedHosts.get(1)); + Assert.assertEquals(host3, prioritizedHosts.get(2)); + } + + @Test + public void prioritizeHostsByHvmCapabilityTestTemplateRequiresHvmShouldNotReorderList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List prioritizedHosts = new ArrayList<>(); + + Mockito.doReturn(true).when(vmTemplateVO).isRequiresHvm(); + firstFitAllocatorSpy.prioritizeHostsByHvmCapability(vmTemplateVO, hostsToCheck, prioritizedHosts); + + Assert.assertEquals(3, prioritizedHosts.size()); + Assert.assertEquals(host1, prioritizedHosts.get(0)); + Assert.assertEquals(host2, prioritizedHosts.get(1)); + Assert.assertEquals(host3, prioritizedHosts.get(2)); + } + + @Test + public void prioritizeHostsWithMatchingGuestOsTestShouldPutMatchingHostInHighPriorityAndHostsThatDoesNotMatchInLowPriorityList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List highPriorityHosts = new ArrayList<>(); + List lowPriorityHosts = new ArrayList<>(); + String guestOsCategory1 = "guestOsCategory1"; + String guestOsCategory2 = "guestOsCategory2"; + + Mockito.doReturn(guestOsCategory1).when(firstFitAllocatorSpy).getTemplateGuestOSCategory(vmTemplateVO); + Mockito.doReturn(guestOsCategory1).when(firstFitAllocatorSpy).getHostGuestOSCategory(host1); + Mockito.doReturn(guestOsCategory2).when(firstFitAllocatorSpy).getHostGuestOSCategory(host2); + Mockito.doReturn(null).when(firstFitAllocatorSpy).getHostGuestOSCategory(host3); + firstFitAllocatorSpy.prioritizeHostsWithMatchingGuestOs(vmTemplateVO,hostsToCheck, highPriorityHosts, lowPriorityHosts); + + Assert.assertEquals(1, highPriorityHosts.size()); + Assert.assertEquals(host1, highPriorityHosts.get(0)); + Assert.assertEquals(1, lowPriorityHosts.size()); + Assert.assertEquals(host2, lowPriorityHosts.get(0)); + } +} From b0b7e3b02f9e264a915341340d861ce6d2246c20 Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Mon, 24 Jun 2024 15:17:15 -0300 Subject: [PATCH 4/7] Move random allocator from cloud-plugins to cloud-server --- plugins/host-allocators/random/pom.xml | 30 ---------------- .../host-allocator-random/module.properties | 18 ---------- .../spring-host-allocator-random-context.xml | 34 ------------------- plugins/pom.xml | 2 -- .../allocator/impl/RandomAllocator.java | 0 .../spring-server-allocator-context.xml | 3 ++ .../allocator/impl/RandomAllocatorTest.java | 0 7 files changed, 3 insertions(+), 84 deletions(-) delete mode 100644 plugins/host-allocators/random/pom.xml delete mode 100644 plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties delete mode 100644 plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml rename {plugins/host-allocators/random => server}/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java (100%) rename {plugins/host-allocators/random => server}/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java (100%) diff --git a/plugins/host-allocators/random/pom.xml b/plugins/host-allocators/random/pom.xml deleted file mode 100644 index f01494914ce6..000000000000 --- a/plugins/host-allocators/random/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - cloud-plugin-host-allocator-random - Apache CloudStack Plugin - Host Allocator Random - - org.apache.cloudstack - cloudstack-plugins - 4.20.0.0-SNAPSHOT - ../../pom.xml - - diff --git a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties b/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties deleted file mode 100644 index dcfe8d3537ff..000000000000 --- a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -name=host-allocator-random -parent=allocator diff --git a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml b/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml deleted file mode 100644 index d84eaafaa5a7..000000000000 --- a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - diff --git a/plugins/pom.xml b/plugins/pom.xml index 279067e2c97f..c9fa36c523d5 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -83,8 +83,6 @@ ha-planners/skip-heurestics - host-allocators/random - hypervisors/baremetal hypervisors/hyperv hypervisors/kvm diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java similarity index 100% rename from plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java rename to server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java diff --git a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml index d3781649845b..28b96b8d194b 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml @@ -34,6 +34,9 @@ + + diff --git a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java similarity index 100% rename from plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java rename to server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java From 81d6a6744b63edd3c05d0f012f6d496f041dad45 Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Thu, 11 Jul 2024 12:46:35 -0300 Subject: [PATCH 5/7] Add BaseAllocator abstract class for duplicate code --- .../com/cloud/deploy/DeploymentPlanner.java | 2 +- .../manager/allocator/impl/BaseAllocator.java | 74 +++++++ .../allocator/impl/FirstFitAllocator.java | 97 ++------- .../allocator/impl/RandomAllocator.java | 53 +---- .../allocator/impl/BaseAllocatorTest.java | 203 ++++++++++++++++++ .../allocator/impl/FirstFitAllocatorTest.java | 160 +------------- .../allocator/impl/RandomAllocatorTest.java | 65 +----- 7 files changed, 311 insertions(+), 343 deletions(-) create mode 100644 server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java create mode 100644 server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java diff --git a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java index 354f9cfaac53..a30856dfc5ad 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java @@ -70,7 +70,7 @@ public interface DeploymentPlanner extends Adapter { boolean canHandle(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid); public enum AllocationAlgorithm { - random, firstfit, userdispersing, userconcentratedpod_random, userconcentratedpod_firstfit; + random, firstfit, userdispersing, userconcentratedpod_random, userconcentratedpod_firstfit, firstfitleastconsumed } public enum PlannerResourceUsage { diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java new file mode 100644 index 000000000000..7aeeaf514976 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java @@ -0,0 +1,74 @@ +package com.cloud.agent.manager.allocator.impl; + +import com.cloud.agent.manager.allocator.HostAllocator; +import com.cloud.capacity.CapacityManager; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.utils.Pair; +import com.cloud.utils.component.AdapterBase; +import org.apache.commons.collections.CollectionUtils; + +import javax.inject.Inject; +import java.util.List; + +public abstract class BaseAllocator extends AdapterBase implements HostAllocator { + + @Inject + protected HostDao hostDao; + + @Inject + protected CapacityManager capacityManager; + + protected void retainHostsMatchingServiceOfferingAndTemplateTags(List availableHosts, Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { + logger.debug("Hosts {} will be checked for template and host tags compatibility.", availableHosts); + + if (offeringHostTag != null) { + logger.debug("Looking for hosts having the tag [{}] specified in the Service Offering.", offeringHostTag); + List hostsWithHostTag = hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag); + logger.debug("Retaining hosts {} because they match the offering host tag {}.", hostsWithHostTag, offeringHostTag); + availableHosts.retainAll(hostsWithHostTag); + } + + if (templateTag != null) { + logger.debug("Looking for hosts having the tag [{}] specified in the Template.", templateTag); + List hostsWithTemplateTag = hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); + logger.debug("Retaining hosts {} because they match the template tag {}.", hostsWithTemplateTag, templateTag); + availableHosts.retainAll(hostsWithTemplateTag); + } + + logger.debug("Remaining hosts after template tag and host tags validations are {}.", availableHosts); + } + + protected void addHostsBasedOnTagRules(String hostTagOnOffering, List clusterHosts) { + List hostsWithTagRules = hostDao.findHostsWithTagRuleThatMatchComputeOfferingTags(hostTagOnOffering); + + if (CollectionUtils.isEmpty(hostsWithTagRules)) { + logger.info("No hosts found with tag rules matching the compute offering tag [{}].", hostTagOnOffering); + return; + } + + logger.info("Found hosts %s with tag rules matching the compute offering tag [{}].", hostsWithTagRules, hostTagOnOffering); + clusterHosts.addAll(hostsWithTagRules); + } + + /** + * Adds hosts with enough CPU capability and enough CPU capacity to the suitable hosts list. + */ + protected boolean hostHasCpuCapabilityAndCapacity(boolean considerReservedCapacity, ServiceOffering offering, Host host) { + logger.debug("Looking for CPU frequency {} MHz and RAM {} MB.", () -> offering.getCpu() * offering.getSpeed(), offering::getRamSize); + Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); + Boolean hasCpuCapability = cpuCapabilityAndCapacity.first(); + Boolean hasCpuCapacity = cpuCapabilityAndCapacity.second(); + + if (hasCpuCapability && hasCpuCapacity) { + logger.debug("Host {} is a suitable host as it has enough CPU capability and CPU capacity.", () -> host); + return true; + } + + logger.debug("Cannot use host {}. Does the host have CPU capability? {}. Does the host have CPU capacity? {}..", () -> host, () -> hasCpuCapability, () -> hasCpuCapacity); + return false; + } + +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 9855548d2f07..f7483e0ae2d6 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -25,16 +25,16 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; -import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.CapacityManager; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; import com.cloud.configuration.Config; @@ -45,7 +45,6 @@ import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostDetailsDao; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; @@ -57,8 +56,6 @@ import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; import com.cloud.user.Account; -import com.cloud.utils.Pair; -import com.cloud.utils.component.AdapterBase; import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.UserVmDetailsDao; @@ -69,9 +66,7 @@ * An allocator that tries to find a fit on a computing host. This allocator does not care whether or not the host supports routing. */ @Component -public class FirstFitAllocator extends AdapterBase implements HostAllocator { - @Inject - protected HostDao _hostDao = null; +public class FirstFitAllocator extends BaseAllocator { @Inject HostDetailsDao _hostDetailsDao = null; @Inject @@ -87,14 +82,13 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { @Inject ServiceOfferingDetailsDao _serviceOfferingDetailsDao; @Inject - CapacityManager _capacityMgr; - @Inject CapacityDao _capacityDao; @Inject UserVmDetailsDao _userVmDetailsDao; boolean _checkHvm = true; - protected String _allocationAlgorithm = "random"; + + protected AllocationAlgorithm allocationAlgorithm = AllocationAlgorithm.random; @Override @@ -146,11 +140,11 @@ protected List retrieveHosts(VirtualMachineProfile vmProfile, Type type, } if (haVmTag != null) { - clusterHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); + clusterHosts.retainAll(hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); } else if (ObjectUtils.allNull(hostTagOnOffering, hostTagOnTemplate)) { clusterHosts.retainAll(_resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId)); } else { - retainHostsMatchingServiceOfferingAndTemplateTags(clusterHosts, hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); + retainHostsMatchingServiceOfferingAndTemplateTags(clusterHosts, type, clusterId, podId, dcId, hostTagOnTemplate, hostTagOnOffering); } filterHostsWithUefiEnabled(type, vmProfile, clusterId, podId, dcId, clusterHosts); @@ -161,41 +155,11 @@ protected List retrieveHosts(VirtualMachineProfile vmProfile, Type type, } - protected void addHostsBasedOnTagRules(String hostTagOnOffering, List clusterHosts) { - List hostsWithTagRules = _hostDao.findHostsWithTagRuleThatMatchComputeOfferingTags(hostTagOnOffering); - - if (CollectionUtils.isEmpty(hostsWithTagRules)) { - logger.info("No hosts found with tag rules matching the compute offering tag [{}].", hostTagOnOffering); - return; - } - - logger.info("Found hosts %s with tag rules matching the compute offering tag [{}].", hostsWithTagRules, hostTagOnOffering); - clusterHosts.addAll(hostsWithTagRules); - } - - protected void retainHostsMatchingServiceOfferingAndTemplateTags(List clusterHosts, String hostTagOnTemplate, String hostTagOnOffering, Type type, Long clusterId, - Long podId, long dcId) { - boolean hasSvcOfferingTag = hostTagOnOffering != null; - boolean hasTemplateTag = hostTagOnTemplate != null; - - if (hasSvcOfferingTag) { - logger.debug("Looking for hosts having the tag [{}] specified in the Service Offering.", hostTagOnOffering); - clusterHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering)); - logger.debug("Hosts with Service Offering tag [{}] are {}.", hostTagOnOffering, clusterHosts); - } - - if (hasTemplateTag) { - logger.debug("Looking for hosts having the tag [{}] specified in the Template.", hostTagOnTemplate); - clusterHosts.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate)); - logger.debug("Hosts with Template tag [{}] are {}.", hostTagOnTemplate, clusterHosts); - } - } - /** * Add all hosts to the avoid set that were not considered during the allocation */ protected void addHostsToAvoidSet(Type type, ExcludeList avoid, Long clusterId, Long podId, long dcId, List suitableHosts) { - List allHostsInCluster = _hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); + List allHostsInCluster = hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); allHostsInCluster.removeAll(suitableHosts); @@ -220,7 +184,7 @@ protected void filterHostsWithUefiEnabled(Type type, VirtualMachineProfile vmPro logger.info("Guest VM is requested with Custom[UEFI] Boot Type enabled."); - List hostsMatchingUefiTag = _hostDao.listByHostCapability(type, clusterId, podId, dcId, Host.HOST_UEFI_ENABLE); + List hostsMatchingUefiTag = hostDao.listByHostCapability(type, clusterId, podId, dcId, Host.HOST_UEFI_ENABLE); logger.debug("Hosts with UEFI enabled are {}.", hostsMatchingUefiTag); clusterHosts.retainAll(hostsMatchingUefiTag); @@ -228,16 +192,16 @@ protected void filterHostsWithUefiEnabled(Type type, VirtualMachineProfile vmPro protected List allocateTo(DeploymentPlan plan, ServiceOffering offering, VMTemplateVO template, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity, Account account) { - switch (_allocationAlgorithm) { - case "random": - case "userconcentratedpod_random": - // Shuffle this so that we don't check the hosts in the same order. + + switch (allocationAlgorithm) { + case random: + case userconcentratedpod_random: Collections.shuffle(hosts); break; - case "userdispersing": + case userdispersing: hosts = reorderHostsByNumberOfVms(plan, hosts, account); break; - case "firstfitleastconsumed": + case firstfitleastconsumed: hosts = reorderHostsByCapacity(plan, hosts); break; } @@ -252,7 +216,6 @@ protected List allocateTo(DeploymentPlan plan, ServiceOffering offering, V return suitableHosts; } - protected List checkHostsCompatibilities(ServiceOffering offering, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { List suitableHosts = new ArrayList<>(); logger.debug("Checking compatibility for the following hosts {}.", suitableHosts); @@ -268,7 +231,7 @@ protected List checkHostsCompatibilities(ServiceOffering offering, Exclude continue; } - if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)) { + if (capacityManager.checkIfHostReachMaxGuestLimit(host)) { logger.debug("Adding host [{}] to the avoid set because this host already has the max number of running (user and/or system) VMs.", () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); avoid.addHost(host.getId()); @@ -279,29 +242,13 @@ protected List checkHostsCompatibilities(ServiceOffering offering, Exclude continue; } - addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(offering, avoid, considerReservedCapacity, host, suitableHosts); - } - return suitableHosts; - } - - - /** - * Adds hosts with CPU capability and CPU capacity to the suitable hosts list. Otherwise, the host is added to the avoid list. - */ - protected void addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(ServiceOffering offering, ExcludeList avoid, boolean considerReservedCapacity, Host host, List suitableHosts) { - logger.debug("Looking for CPU frequency {} MHz and RAM {} MB.", () -> offering.getCpu() * offering.getSpeed(), () -> offering.getRamSize()); - Pair cpuCapabilityAndCapacity = _capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - Boolean hasCpuCapability = cpuCapabilityAndCapacity.first(); - Boolean hasCpuCapacity = cpuCapabilityAndCapacity.second(); - - if (hasCpuCapability && hasCpuCapacity) { - logger.debug("Found a suitable host, adding to list host [{}].", () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", "name")); - suitableHosts.add(host); - } else { - logger.debug("Not using host {}; host has cpu capability? {}, host has capacity? {}.", () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(host, "uuid", - "name"), () -> hasCpuCapability, () -> hasCpuCapacity); + if (hostHasCpuCapabilityAndCapacity(considerReservedCapacity,offering, host)) { + suitableHosts.add(host); + continue; + } avoid.addHost(host.getId()); } + return suitableHosts; } protected boolean offeringRequestedVGpuAndHostDoesNotHaveIt(ServiceOffering offering, ExcludeList avoid, Host host) { @@ -557,7 +504,7 @@ public boolean configure(String name, Map params) throws Configu String allocationAlgorithm = configs.get("vm.allocation.algorithm"); if (allocationAlgorithm != null) { - _allocationAlgorithm = allocationAlgorithm; + this.allocationAlgorithm = EnumUtils.getEnum(AllocationAlgorithm.class, allocationAlgorithm); } String value = configs.get("xenserver.check.hvm"); _checkHvm = value == null || Boolean.parseBoolean(value); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java index 9f6e8dde8d55..499bf5b5980f 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -23,33 +23,23 @@ import javax.inject.Inject; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.ListUtils; import org.apache.commons.lang3.ObjectUtils; import org.springframework.stereotype.Component; -import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.CapacityManager; import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; import com.cloud.storage.VMTemplateVO; -import com.cloud.utils.Pair; -import com.cloud.utils.component.AdapterBase; import com.cloud.vm.VirtualMachineProfile; @Component -public class RandomAllocator extends AdapterBase implements HostAllocator { - @Inject - private HostDao _hostDao; +public class RandomAllocator extends BaseAllocator { @Inject private ResourceManager _resourceMgr; - @Inject - private CapacityManager capacityManager; protected List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { @@ -103,21 +93,6 @@ protected List filterAvailableHosts(ExcludeList avoid, int returnUpTo, boo return suitableHosts; } - - protected boolean hostHasCpuCapabilityAndCapacity(boolean considerReservedCapacity, ServiceOffering offering, Host host) { - Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - Boolean hasCpuCapability = cpuCapabilityAndCapacity.first(); - Boolean hasCpuCapacity = cpuCapabilityAndCapacity.second(); - - if (hasCpuCapability && hasCpuCapacity) { - logger.debug("Host {} has enough CPU capability and CPU capacity.", host); - return true; - } - - logger.debug("Not using host [{}]. Does the host have cpu capability? {}. Does the host have capacity? {}.", () -> host, () -> hasCpuCapability, () -> hasCpuCapacity); - return false; - } - /** * @return all computing hosts, regardless of whether they support routing. */ @@ -132,38 +107,18 @@ protected List retrieveHosts(Type type, List hosts, VMTemplateVO } if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - retainHostsWithMatchingTags(availableHosts, type, clusterId, podId, dcId, offeringHostTag, templateTag); + retainHostsMatchingServiceOfferingAndTemplateTags(availableHosts, type, clusterId, podId, dcId, offeringHostTag, templateTag); } else { - List hostsWithNoRuleTag = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); + List hostsWithNoRuleTag = hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); logger.debug("Retaining hosts {} because they do not have rule tags.", hostsWithNoRuleTag); availableHosts.retainAll(hostsWithNoRuleTag); } - List hostsWithTagRuleThatMatchComputeOfferingTags = _hostDao.findHostsWithTagRuleThatMatchComputeOfferingTags(offeringHostTag); - logger.debug("Adding hosts {} to the available pool hosts because they match the compute offering's tag.", hostsWithTagRuleThatMatchComputeOfferingTags, offeringHostTag); - availableHosts = ListUtils.union(availableHosts, hostsWithTagRuleThatMatchComputeOfferingTags); + addHostsBasedOnTagRules(offeringHostTag, availableHosts); return availableHosts; } - protected void retainHostsWithMatchingTags(List availableHosts, Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { - logger.debug("Hosts {} will be checked for template and host tags compatibility.", availableHosts); - - if (offeringHostTag != null) { - List hostsWithHostTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag); - logger.debug("Retaining hosts {} because they match the offering host tag {}.", hostsWithHostTag, offeringHostTag); - availableHosts.retainAll(hostsWithHostTag); - } - - if (templateTag != null) { - List hostsWithTemplateTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); - logger.debug("Retaining hosts {} because they match the template tag {}.", hostsWithTemplateTag, offeringHostTag); - availableHosts.retainAll(hostsWithTemplateTag); - } - - logger.debug("Remaining hosts after template tag and host tags validations are {}.", availableHosts); - } - @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java new file mode 100644 index 000000000000..14f95755d0b2 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java @@ -0,0 +1,203 @@ +package com.cloud.agent.manager.allocator.impl; + +import com.cloud.capacity.CapacityManager; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.utils.Pair; +import com.cloud.vm.VirtualMachineProfile; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@RunWith(MockitoJUnitRunner.class) +public class BaseAllocatorTest { + @Mock + HostDao hostDaoMock; + + @Mock + CapacityManager capacityManagerMock; + + @InjectMocks + @Spy + BaseAllocator baseAllocator = new MockBaseAllocator(); + + private final Host.Type type = Host.Type.Routing; + + private final Long clusterId = 1L; + + private final Long podId = 1L; + + private final Long dcId = 1L; + + private final HostVO host1 = Mockito.mock(HostVO.class); + + private final HostVO host2 = Mockito.mock(HostVO.class); + + private final HostVO host3 = Mockito.mock(HostVO.class); + + private final ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + + private final String hostTag = "hostTag"; + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagShouldRetainHostsWithServiceOfferingTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingTags = new ArrayList<>(Arrays.asList(host1, host3)); + String hostTagOnTemplate = "hostTagOnTemplate"; + String hostTagOnOffering = null; + + Mockito.doReturn(hostsWithMathingTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, clusterId, podId, dcId, hostTagOnTemplate, hostTagOnOffering); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagAndHasHostTagOnTemplateShouldRetainHostsWithServiceOfferingTagAndTemplateTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); + List hostsWithMathingTemplateTags = new ArrayList<>(Arrays.asList(host1, host2)); + String hostTagOnTemplate = "hostTagOnTemplate"; + String hostTagOnOffering = "hostTagOnOffering"; + + Mockito.doReturn(hostsWithMathingTemplateTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, clusterId, podId, dcId, hostTagOnTemplate, hostTagOnOffering); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasHostTagOnTemplateShouldRetainHostsWithTemplateTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); + String hostTagOnTemplate = null; + String hostTagOnOffering = "hostTagOnOffering"; + + Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, clusterId, podId, dcId, hostTagOnTemplate, hostTagOnOffering); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestNoServiceTagAndNoTemplateTagShouldHaveAllSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + String hostTagOnTemplate = null; + String hostTagOnOffering = null; + + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, clusterId, podId, dcId, hostTagOnTemplate, hostTagOnOffering); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsEmptyShouldNotAddToSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List emptyList = new ArrayList<>(); + + Mockito.doReturn(emptyList).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); + baseAllocator.addHostsBasedOnTagRules(hostTag, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + } + + @Test + public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsNotEmptyShouldAddToSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsMatchingRuleTag = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(hostsMatchingRuleTag).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); + baseAllocator.addHostsBasedOnTagRules(hostTag, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityAndCpuCapacityShouldReturnTrue() { + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertTrue(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityButNoCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityButHasCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityAndCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + class MockBaseAllocator extends BaseAllocator { + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Host.Type type, DeploymentPlanner.ExcludeList avoid, int returnUpTo) { + return null; + } + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Host.Type type, DeploymentPlanner.ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { + return null; + } + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java index 3921fa028bef..92e9ca10cbd1 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java @@ -17,7 +17,6 @@ package com.cloud.agent.manager.allocator.impl; import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.CapacityManager; import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner; import com.cloud.host.Host; @@ -30,7 +29,6 @@ import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.VMTemplateVO; import com.cloud.user.Account; -import com.cloud.utils.Pair; import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.UserVmDetailsDao; @@ -60,9 +58,6 @@ public class FirstFitAllocatorTest { @Mock UserVmDetailsDao userVmDetailsDaoMock; - @Mock - CapacityManager capacityManagerMock; - @Mock ServiceOfferingDetailsDao serviceOfferingDetailsDao; @@ -215,98 +210,12 @@ public void retrieveHostsTestHostsToFilterIsNullAndNoHaTagWithHostTagShouldCallR List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); - Mockito.doNothing().when(firstFitAllocatorSpy).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.anyString(), Mockito.anyString(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doNothing().when(firstFitAllocatorSpy).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, hostTag, templateTag); - Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.anyString(), Mockito.anyString(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); - } - - @Test - public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsEmptyShouldNotAddToSuitableHosts() { - List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); - - Mockito.doReturn(emptyList).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); - firstFitAllocatorSpy.addHostsBasedOnTagRules(hostTag, suitableHosts); - - Assert.assertEquals(2, suitableHosts.size()); - Assert.assertEquals(host1, suitableHosts.get(0)); - Assert.assertEquals(host2, suitableHosts.get(1)); - } - - @Test - public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsNotEmptyShouldAddToSuitableHosts() { - List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); - List hostsMatchingRuleTag = new ArrayList<>(Arrays.asList(host3)); - - Mockito.doReturn(hostsMatchingRuleTag).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); - firstFitAllocatorSpy.addHostsBasedOnTagRules(hostTag, suitableHosts); - - Assert.assertEquals(3, suitableHosts.size()); - Assert.assertEquals(host1, suitableHosts.get(0)); - Assert.assertEquals(host2, suitableHosts.get(1)); - Assert.assertEquals(host3, suitableHosts.get(2)); - } - - @Test - public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagShouldRetainHostsWithServiceOfferingTag() { - List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); - List hostsWithMathingTags = new ArrayList<>(Arrays.asList(host1, host3)); - String hostTagOnTemplate = "hostTagOnTemplate"; - String hostTagOnOffering = null; - - Mockito.doReturn(hostsWithMathingTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); - firstFitAllocatorSpy.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); - - Assert.assertEquals(2, suitableHosts.size()); - Assert.assertEquals(host1, suitableHosts.get(0)); - Assert.assertEquals(host3, suitableHosts.get(1)); - } - - @Test - public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagAndHasHostTagOnTemplateShouldRetainHostsWithServiceOfferingTagAndTemplateTag() { - List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); - List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); - List hostsWithMathingTemplateTags = new ArrayList<>(Arrays.asList(host1, host2)); - String hostTagOnTemplate = "hostTagOnTemplate"; - String hostTagOnOffering = "hostTagOnOffering"; - - Mockito.doReturn(hostsWithMathingTemplateTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); - Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); - firstFitAllocatorSpy.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); - - Assert.assertEquals(1, suitableHosts.size()); - Assert.assertEquals(host1, suitableHosts.get(0)); - } - - @Test - public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasHostTagOnTemplateShouldRetainHostsWithTemplateTag() { - List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); - List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); - String hostTagOnTemplate = null; - String hostTagOnOffering = "hostTagOnOffering"; - - Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); - firstFitAllocatorSpy.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); - - Assert.assertEquals(2, suitableHosts.size()); - Assert.assertEquals(host1, suitableHosts.get(0)); - Assert.assertEquals(host3, suitableHosts.get(1)); - } - - @Test - public void retainHostsMatchingServiceOfferingAndTemplateTagsTestNoServiceTagAndNoTemplateTagShouldHaveAllSuitableHosts() { - List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); - String hostTagOnTemplate = null; - String hostTagOnOffering = null; - - firstFitAllocatorSpy.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, hostTagOnTemplate, hostTagOnOffering, type, clusterId, podId, dcId); - - Assert.assertEquals(3, suitableHosts.size()); - Assert.assertEquals(host1, suitableHosts.get(0)); - Assert.assertEquals(host2, suitableHosts.get(1)); - Assert.assertEquals(host3, suitableHosts.get(2)); + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); } @Test @@ -398,71 +307,6 @@ public void filterHostsWithUefiEnabledTestDetailWithUefiWithSecureModeShouldFilt Assert.assertEquals(host3, suitableHosts.get(1)); } - @Test - public void addHostToSuitableHostIfHasCpuCapacityAndCpuCapabilityTestHostHasCapacityAndCapabilityShouldBeAddedToSuitableHosts() { - List suitableHosts = new ArrayList<>(); - Boolean hasCpuCapability = true; - Boolean hasCpuCapacity = true; - Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); - - Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); - firstFitAllocatorSpy.addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(serviceOffering, excludeList, considerReservedCapacity, host1, suitableHosts); - - Assert.assertEquals(1, suitableHosts.size()); - Assert.assertEquals(host1, suitableHosts.get(0)); - } - - @Test - public void addHostToSuitableHostIfHasCpuCapacityAndCpuCapabilityTestHostDoesNotHaveCapacityAndCapabilityShouldBeAddedToTheAvoidList() { - List suitableHosts = new ArrayList<>(); - Boolean hasCpuCapability = true; - Boolean hasCpuCapacity = false; - Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); - - Mockito.doReturn(1L).when(host1).getId(); - Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); - Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); - Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); - firstFitAllocatorSpy.addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(serviceOffering, excludeList, considerReservedCapacity, host1, suitableHosts); - - Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); - Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); - } - - @Test - public void addHostToSuitableHostIfHasCpuCapacityAndCpuCapabilityTestHostHasCapacityButNoCapabilityShouldBeAddedToTheAvoidList() { - List suitableHosts = new ArrayList<>(); - Boolean hasCpuCapability = false; - Boolean hasCpuCapacity = true; - Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); - - Mockito.doReturn(1L).when(host1).getId(); - Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); - Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); - Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); - firstFitAllocatorSpy.addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(serviceOffering, excludeList, considerReservedCapacity, host1, suitableHosts); - - Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); - Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); - } - - @Test - public void addHostToSuitableHostIfHasCpuCapacityAndCpuCapabilityTestHostDoesNotHaveCapacityNoCapabilityShouldBeAddedToTheAvoidList() { - List suitableHosts = new ArrayList<>(); - Boolean hasCpuCapability = false; - Boolean hasCpuCapacity = false; - Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); - - Mockito.doReturn(1L).when(host1).getId(); - Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); - Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); - Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); - firstFitAllocatorSpy.addHostToSuitableHostIfHasCpuCapacityAndCpuCapability(serviceOffering, excludeList, considerReservedCapacity, host1, suitableHosts); - - Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); - Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); - } - @Test public void offeringRequestedVGpuAndHostDoesNotHaveItTestNoVGpuRequestedShouldReturnFalse() { ServiceOfferingDetailsVO requestedVGpuType = null; diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java index a16a6391d8d3..c00fa43b44aa 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java @@ -21,14 +21,12 @@ import java.util.List; import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.CapacityManager; import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; import com.cloud.service.ServiceOfferingVO; import com.cloud.storage.VMTemplateVO; -import com.cloud.utils.Pair; import com.cloud.vm.VirtualMachineProfile; import org.apache.commons.collections.CollectionUtils; import org.junit.Assert; @@ -57,9 +55,6 @@ public class RandomAllocatorTest { @Mock ResourceManager resourceManagerMock; - @Mock - CapacityManager capacityManagerMock; - private final Host.Type type = Host.Type.Routing; private final Long clusterId = 1L; @@ -72,8 +67,6 @@ public class RandomAllocatorTest { private final String hostTag = "hostTag"; - private final String templateTag = "templateTag"; - private final HostVO host1 = Mockito.mock(HostVO.class); private final HostVO host2 = Mockito.mock(HostVO.class); @@ -106,32 +99,32 @@ public void testListHostsByTags() { // No template tagged host ArrayList noTemplateTaggedHosts = new ArrayList<>(Arrays.asList(host1, host2)); Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(new ArrayList<>()); - randomAllocator.retainHostsWithMatchingTags(noTemplateTaggedHosts, type, id, id, id, offeringTag, templateTag); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noTemplateTaggedHosts, type, id, id, id, offeringTag, templateTag); Assert.assertTrue(CollectionUtils.isEmpty(noTemplateTaggedHosts)); // Different template tagged host ArrayList differentTemplateTaggedHost = new ArrayList<>(Arrays.asList(host1, host2)); HostVO host3 = Mockito.mock(HostVO.class); Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host3)); - randomAllocator.retainHostsWithMatchingTags(differentTemplateTaggedHost, type, id, id, id, offeringTag, templateTag); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(differentTemplateTaggedHost, type, id, id, id, offeringTag, templateTag); Assert.assertTrue(CollectionUtils.isEmpty(differentTemplateTaggedHost)); // Matching template tagged host ArrayList matchingTemplateTaggedHost = new ArrayList<>(Arrays.asList(host1, host2)); Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host1)); - randomAllocator.retainHostsWithMatchingTags(matchingTemplateTaggedHost, type, id, id, id, offeringTag, templateTag); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(matchingTemplateTaggedHost, type, id, id, id, offeringTag, templateTag); Assert.assertFalse(CollectionUtils.isEmpty(matchingTemplateTaggedHost)); Assert.assertEquals(1, matchingTemplateTaggedHost.size()); // No template tag ArrayList noTemplateTag = new ArrayList<>(Arrays.asList(host1, host2)); - randomAllocator.retainHostsWithMatchingTags(noTemplateTag, type, id, id, id, offeringTag, null); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noTemplateTag, type, id, id, id, offeringTag, null); Assert.assertFalse(CollectionUtils.isEmpty(noTemplateTag)); Assert.assertEquals(2, noTemplateTag.size()); // No offering tag ArrayList noOfferingTag = new ArrayList<>(Arrays.asList(host1, host2)); - randomAllocator.retainHostsWithMatchingTags(noOfferingTag, type, id, id, id, null, templateTag); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noOfferingTag, type, id, id, id, null, templateTag); Assert.assertFalse(CollectionUtils.isEmpty(noOfferingTag)); Assert.assertEquals(1, noOfferingTag.size()); } @@ -221,54 +214,6 @@ public void filterAvailableHostsTestOnlyHost2HasCpuCapacityAndCapabilityShouldRe Assert.assertEquals(host2, suitableHosts.get(0)); } - @Test - public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityAndCpuCapacityShouldReturnTrue() { - Boolean hasCpuCapability = true; - Boolean hasCpuCapacity = true; - Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); - - Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); - boolean result = randomAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); - - Assert.assertTrue(result); - } - - @Test - public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityButNoCpuCapacityShouldReturnFalse() { - Boolean hasCpuCapability = true; - Boolean hasCpuCapacity = false; - Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); - - Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); - boolean result = randomAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); - - Assert.assertFalse(result); - } - - @Test - public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityButHasCpuCapacityShouldReturnFalse() { - Boolean hasCpuCapability = false; - Boolean hasCpuCapacity = true; - Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); - - Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); - boolean result = randomAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); - - Assert.assertFalse(result); - } - - @Test - public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityAndCpuCapacityShouldReturnFalse() { - Boolean hasCpuCapability = false; - Boolean hasCpuCapacity = false; - Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); - - Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); - boolean result = randomAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); - - Assert.assertFalse(result); - } - @Test public void retrieveHostsTestProvidedHostsNullAndNoHostTagAndNoTagRuleShouldOnlyReturnHostsWithNoTags() { List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); From 00bc7f95938e688121ac1726e830a0b818658b64 Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Thu, 11 Jul 2024 12:50:10 -0300 Subject: [PATCH 6/7] Add missing license --- .../manager/allocator/impl/BaseAllocator.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java index 7aeeaf514976..a114b822e868 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package com.cloud.agent.manager.allocator.impl; import com.cloud.agent.manager.allocator.HostAllocator; From 952c273715007ac1c17ff91f93e78518ff794b55 Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Thu, 11 Jul 2024 12:54:06 -0300 Subject: [PATCH 7/7] Add missing license to unit test file --- .../allocator/impl/BaseAllocatorTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java index 14f95755d0b2..71fbc83ca705 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package com.cloud.agent.manager.allocator.impl; import com.cloud.capacity.CapacityManager;