From bd8f446eaa17a35d1b50e6206f82374ca1600125 Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Thu, 21 Sep 2017 19:20:37 -0700 Subject: [PATCH 1/4] YARN-2162 --- .../fair/AllocationConfiguration.java | 44 ++++------ .../fair/AllocationFileLoaderService.java | 47 ++++++----- .../scheduler/fair/ConfigurableResource.java | 59 ++++++++++++++ .../scheduler/fair/FSLeafQueue.java | 6 +- .../scheduler/fair/FSParentQueue.java | 6 +- .../scheduler/fair/FSQueue.java | 33 +++++--- .../scheduler/fair/FairScheduler.java | 3 +- .../fair/FairSchedulerConfiguration.java | 51 ++++++++++-- .../scheduler/fair/QueueManager.java | 3 +- .../fair/TestAllocationFileLoaderService.java | 22 ++--- .../fair/TestConfigurableResource.java | 62 ++++++++++++++ .../scheduler/fair/TestFSLeafQueue.java | 2 +- .../scheduler/fair/TestFSParentQueue.java | 3 + .../scheduler/fair/TestFairScheduler.java | 15 ++-- .../fair/TestFairSchedulerConfiguration.java | 81 +++++++++++++++---- .../fair/TestMaxRunningAppsEnforcer.java | 5 +- .../scheduler/fair/TestQueueManager.java | 5 +- .../dao/TestFairSchedulerQueueInfo.java | 3 + .../src/site/markdown/FairScheduler.md | 4 +- 19 files changed, 345 insertions(+), 109 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 7bd6959485689..f8aaf7de0346c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -40,15 +40,13 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*"); private static final AccessControlList NOBODY_ACL = new AccessControlList(" "); - private static final ResourceCalculator RESOURCE_CALCULATOR = - new DefaultResourceCalculator(); // Minimum resource allocation for each queue private final Map minQueueResources; // Maximum amount of resources per queue @VisibleForTesting - final Map maxQueueResources; + final Map maxQueueResources; // Maximum amount of resources for each queue's ad hoc children - private final Map maxChildQueueResources; + private final Map maxChildQueueResources; // Sharing weights for each queue private final Map queueWeights; @@ -60,7 +58,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { final Map userMaxApps; private final int userMaxAppsDefault; private final int queueMaxAppsDefault; - private final Resource queueMaxResourcesDefault; + private final ConfigurableResource queueMaxResourcesDefault; // Maximum resource share for each leaf queue that can be used to run AMs final Map queueMaxAMShares; @@ -109,14 +107,14 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { private final Set nonPreemptableQueues; public AllocationConfiguration(Map minQueueResources, - Map maxQueueResources, - Map maxChildQueueResources, + Map maxQueueResources, + Map maxChildQueueResources, Map queueMaxApps, Map userMaxApps, Map queueWeights, Map queueMaxAMShares, int userMaxAppsDefault, int queueMaxAppsDefault, - Resource queueMaxResourcesDefault, + ConfigurableResource queueMaxResourcesDefault, float queueMaxAMShareDefault, Map schedulingPolicies, SchedulingPolicy defaultSchedulingPolicy, @@ -165,7 +163,7 @@ public AllocationConfiguration(Configuration conf) { queueMaxAMShares = new HashMap<>(); userMaxAppsDefault = Integer.MAX_VALUE; queueMaxAppsDefault = Integer.MAX_VALUE; - queueMaxResourcesDefault = Resources.unbounded(); + queueMaxResourcesDefault = new ConfigurableResource(Resources.unbounded()); queueMaxAMShareDefault = 0.5f; queueAcls = new HashMap<>(); resAcls = new HashMap<>(); @@ -291,26 +289,17 @@ Resource getMinResources(String queue) { /** * Get the maximum resource allocation for the given queue. If the max in not - * set, return the larger of the min and the default max. + * set, return the default max. * * @param queue the target queue's name * @return the max allocation on this queue */ - @VisibleForTesting - Resource getMaxResources(String queue) { - Resource maxQueueResource = maxQueueResources.get(queue); + @VisibleForTesting ConfigurableResource getMaxResources(String queue) { + ConfigurableResource maxQueueResource = maxQueueResources.get(queue); if (maxQueueResource == null) { - Resource minQueueResource = minQueueResources.get(queue); - if (minQueueResource != null && - Resources.greaterThan(RESOURCE_CALCULATOR, Resources.unbounded(), - minQueueResource, queueMaxResourcesDefault)) { - return minQueueResource; - } else { - return queueMaxResourcesDefault; - } - } else { - return maxQueueResource; + maxQueueResource = queueMaxResourcesDefault; } + return maxQueueResource; } /** @@ -319,8 +308,7 @@ Resource getMaxResources(String queue) { * @param queue the target queue's name * @return the max allocation on this queue or null if not set */ - @VisibleForTesting - Resource getMaxChildResources(String queue) { + @VisibleForTesting ConfigurableResource getMaxChildResources(String queue) { return maxChildQueueResources.get(queue); } @@ -418,9 +406,9 @@ public void initFSQueue(FSQueue queue){ queue.setMaxChildQueueResource(getMaxChildResources(name)); // Set queue metrics. - queue.getMetrics().setMinShare(getMinResources(name)); - queue.getMetrics().setMaxShare(getMaxResources(name)); - queue.getMetrics().setMaxApps(getQueueMaxApps(name)); + queue.getMetrics().setMinShare(queue.getMinShare()); + queue.getMetrics().setMaxShare(queue.getMaxShare()); + queue.getMetrics().setMaxApps(queue.getMaxRunningApps()); queue.getMetrics().setSchedulingPolicy(getSchedulingPolicy(name).getName()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index 4d918c1d6f4c7..d58abd4f6b984 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -226,8 +226,8 @@ public synchronized void reloadAllocations() throws IOException, // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map minQueueResources = new HashMap<>(); - Map maxQueueResources = new HashMap<>(); - Map maxChildQueueResources = new HashMap<>(); + Map maxQueueResources = new HashMap<>(); + Map maxChildQueueResources = new HashMap<>(); Map queueMaxApps = new HashMap<>(); Map userMaxApps = new HashMap<>(); Map queueMaxAMShares = new HashMap<>(); @@ -244,7 +244,8 @@ public synchronized void reloadAllocations() throws IOException, Set nonPreemptableQueues = new HashSet<>(); int userMaxAppsDefault = Integer.MAX_VALUE; int queueMaxAppsDefault = Integer.MAX_VALUE; - Resource queueMaxResourcesDefault = Resources.unbounded(); + ConfigurableResource queueMaxResourcesDefault = + new ConfigurableResource(Resources.unbounded()); float queueMaxAMShareDefault = 0.5f; long defaultFairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; @@ -305,7 +306,7 @@ public synchronized void reloadAllocations() throws IOException, } } else if ("queueMaxResourcesDefault".equals(element.getTagName())) { String text = ((Text)element.getFirstChild()).getData().trim(); - Resource val = + ConfigurableResource val = FairSchedulerConfiguration.parseResourceConfigValue(text); queueMaxResourcesDefault = val; } else if ("userMaxAppsDefault".equals(element.getTagName())) { @@ -448,8 +449,8 @@ public synchronized void reloadAllocations() throws IOException, */ private void loadQueue(String parentName, Element element, Map minQueueResources, - Map maxQueueResources, - Map maxChildQueueResources, + Map maxQueueResources, + Map maxChildQueueResources, Map queueMaxApps, Map userMaxApps, Map queueMaxAMShares, @@ -496,17 +497,17 @@ private void loadQueue(String parentName, Element element, Element field = (Element) fieldNode; if ("minResources".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); - Resource val = + ConfigurableResource val = FairSchedulerConfiguration.parseResourceConfigValue(text); - minQueueResources.put(queueName, val); + minQueueResources.put(queueName, val.getResource()); } else if ("maxResources".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); - Resource val = + ConfigurableResource val = FairSchedulerConfiguration.parseResourceConfigValue(text); maxQueueResources.put(queueName, val); } else if ("maxChildResources".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); - Resource val = + ConfigurableResource val = FairSchedulerConfiguration.parseResourceConfigValue(text); maxChildQueueResources.put(queueName, val); } else if ("maxRunningApps".equals(field.getTagName())) { @@ -606,14 +607,24 @@ private void loadQueue(String parentName, Element element, queueAcls.put(queueName, acls); resAcls.put(queueName, racls); - if (maxQueueResources.containsKey(queueName) && - minQueueResources.containsKey(queueName) - && !Resources.fitsIn(minQueueResources.get(queueName), - maxQueueResources.get(queueName))) { - LOG.warn( - String.format("Queue %s has max resources %s less than " - + "min resources %s", queueName, maxQueueResources.get(queueName), - minQueueResources.get(queueName))); + checkMinAndMaxResource(minQueueResources, maxQueueResources, queueName); + } + + private void checkMinAndMaxResource(Map minQueueResources, + Map maxQueueResources, String queueName) { + + if (maxQueueResources.containsKey(queueName) + && minQueueResources.containsKey(queueName)) { + + Resource maxResource = maxQueueResources.get(queueName).getResource(); + Resource minResource = minQueueResources.get(queueName); + + // check whether max resource is bigger or equals to min resource when max + // resource are absolute values + if (maxResource != null && !Resources.fitsIn(minResource, maxResource)) { + LOG.warn(String.format("Queue %s has max resources %s less than " + + "min resources %s", queueName, maxResource, minResource)); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java new file mode 100644 index 0000000000000..fffcb96e5cd95 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.Resource; + +/** + * A {@code ConfigurableResource} object represents an entity that is used to + * configure resources, such as maximum resources of a queue. It can be + * percentage of cluster resources or an absolute value. + */ +@Private +@Unstable +public class ConfigurableResource { + private final Resource resource; + private final double[] percentages; + + public ConfigurableResource(double[] percentages) { + this.percentages = percentages.clone(); + this.resource = null; + } + + public ConfigurableResource(Resource resource) { + this.percentages = null; + this.resource = resource; + } + + public Resource getResource(Resource clusterResource) { + if (percentages != null && clusterResource != null) { + long memory = (long) (clusterResource.getMemorySize() * percentages[0]); + int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]); + return Resource.newInstance(memory, vcore); + } else { + return resource; + } + } + + public Resource getResource() { + return resource; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 89ad1e69643bc..60477d4e093d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -332,10 +332,10 @@ public void updateDemand() { readLock.unlock(); } // Cap demand to maxShare to limit allocation to maxShare - demand = Resources.componentwiseMin(tmpDemand, maxShare); + demand = Resources.componentwiseMin(tmpDemand, getMaxShare()); if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand - + "; the max is " + maxShare); + + "; the max is " + getMaxShare()); LOG.debug("The updated fairshare for " + getName() + " is " + getFairShare()); } @@ -611,7 +611,7 @@ protected void dumpStateInternal(StringBuilder sb) { ", Policy: " + policy.getName() + ", FairShare: " + getFairShare() + ", SteadyFairShare: " + getSteadyFairShare() + - ", MaxShare: " + maxShare + + ", MaxShare: " + getMaxShare() + ", MinShare: " + minShare + ", ResourceUsage: " + getResourceUsage() + ", Demand: " + getDemand() + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index 5b4e4dc9e5e92..e42c9f53f411a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -150,13 +150,13 @@ public void updateDemand() { } } // Cap demand to maxShare to limit allocation to maxShare - demand = Resources.componentwiseMin(demand, maxShare); + demand = Resources.componentwiseMin(demand, getMaxShare()); } finally { writeLock.unlock(); } if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand + - "; the max is " + maxShare); + "; the max is " + getMaxShare()); } } @@ -300,7 +300,7 @@ protected void dumpStateInternal(StringBuilder sb) { ", Policy: " + policy.getName() + ", FairShare: " + getFairShare() + ", SteadyFairShare: " + getSteadyFairShare() + - ", MaxShare: " + maxShare + + ", MaxShare: " + getMaxShare() + ", MinShare: " + minShare + ", ResourceUsage: " + getResourceUsage() + ", Demand: " + getDemand() + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 8ae3cb6949600..e0df480dbd1da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -71,9 +71,9 @@ public abstract class FSQueue implements Queue, Schedulable { protected float weights; protected Resource minShare; - protected Resource maxShare; + private ConfigurableResource maxShare; protected int maxRunningApps; - protected Resource maxChildQueueResource; + private ConfigurableResource maxChildQueueResource; // maxAMShare is a value between 0 and 1. protected float maxAMShare; @@ -105,7 +105,7 @@ public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { * * @param recursive whether child queues should be reinitialized recursively */ - public void reinit(boolean recursive) { + public final void reinit(boolean recursive) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); allocConf.initFSQueue(this); updatePreemptionVariables(); @@ -157,26 +157,35 @@ public Resource getMinShare() { return minShare; } - public void setMaxShare(Resource maxShare){ + public void setMaxShare(ConfigurableResource maxShare){ this.maxShare = maxShare; } + @Override + public Resource getMaxShare() { + Resource maxResource = maxShare.getResource(scheduler.getClusterResource()); + + // Max resource should be greater than or equal to min resource + Resource result = Resources.componentwiseMax(maxResource, minShare); + + if (!Resources.equals(maxResource, result)) { + LOG.warn(String.format("Queue %s has max resources %s less than " + + "min resources %s", getName(), maxResource, minShare)); + } + return result; + } + public Resource getReservedResource() { reservedResource.setMemorySize(metrics.getReservedMB()); reservedResource.setVirtualCores(metrics.getReservedVirtualCores()); return reservedResource; } - @Override - public Resource getMaxShare() { - return maxShare; - } - - public void setMaxChildQueueResource(Resource maxChildShare){ + public void setMaxChildQueueResource(ConfigurableResource maxChildShare){ this.maxChildQueueResource = maxChildShare; } - public Resource getMaxChildQueueResource() { + public ConfigurableResource getMaxChildQueueResource() { return maxChildQueueResource; } @@ -415,7 +424,7 @@ boolean assignContainerPreCheck(FSSchedulerNode node) { + " because it has reserved containers."); } return false; - } else if (!Resources.fitsIn(getResourceUsage(), maxShare)) { + } else if (!Resources.fitsIn(getResourceUsage(), getMaxShare())) { if (LOG.isDebugEnabled()) { LOG.debug("Assigning container failed on node '" + node.getNodeName() + " because queue resource usage is larger than MaxShare: " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 9cf2b2e764dca..37f583ec7dbd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -1513,7 +1513,8 @@ private void applyChildDefaults() { if ((queue.getParent() != null) && !configuredLeafQueues.contains(queue.getName()) && !configuredParentQueues.contains(queue.getName())) { - Resource max = queue.getParent().getMaxChildQueueResource(); + ConfigurableResource max = queue.getParent(). + getMaxChildQueueResource(); if (max != null) { queue.setMaxShare(max); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 8e8e37b2dbf8f..1abde8424c655 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -29,6 +29,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.Resources; @@ -287,19 +288,59 @@ public float getReservableNodes() { * * @throws AllocationConfigurationException */ - public static Resource parseResourceConfigValue(String val) + public static ConfigurableResource parseResourceConfigValue(String val) throws AllocationConfigurationException { + ConfigurableResource configurableResource; try { val = StringUtils.toLowerCase(val); - int memory = findResource(val, "mb"); - int vcores = findResource(val, "vcores"); - return BuilderUtils.newResource(memory, vcores); + if (val.contains("%")) { + configurableResource = new ConfigurableResource( + getResourcePercentage(val)); + } else { + int memory = findResource(val, "mb"); + int vcores = findResource(val, "vcores"); + configurableResource = new ConfigurableResource( + BuilderUtils.newResource(memory, vcores)); + } } catch (AllocationConfigurationException ex) { throw ex; } catch (Exception ex) { throw new AllocationConfigurationException( "Error reading resource config", ex); } + return configurableResource; + } + + private static double[] getResourcePercentage( + String val) throws AllocationConfigurationException { + double[] resourcePercentage = new double[ResourceType.values().length]; + String[] strings = val.split(","); + if (strings.length == 1) { + double percentage = findPercentage(strings[0], ""); + for (int i = 0; i < ResourceType.values().length; i++) { + resourcePercentage[i] = percentage/100; + } + } else { + resourcePercentage[0] = findPercentage(val, "memory")/100; + resourcePercentage[1] = findPercentage(val, "cpu")/100; + } + return resourcePercentage; + } + + private static double findPercentage(String val, String units) + throws AllocationConfigurationException { + Pattern pattern = Pattern.compile("((\\d+)(\\.\\d*)?)\\s*%\\s*" + units); + Matcher matcher = pattern.matcher(val); + if (!matcher.find()) { + if (units.equals("")) { + throw new AllocationConfigurationException("Invalid percentage: " + + val); + } else { + throw new AllocationConfigurationException("Missing resource: " + + units); + } + } + return Double.parseDouble(matcher.group(1)); } public long getUpdateInterval() { @@ -307,7 +348,7 @@ public long getUpdateInterval() { } private static int findResource(String val, String units) - throws AllocationConfigurationException { + throws AllocationConfigurationException { Pattern pattern = Pattern.compile("(\\d+)(\\.\\d*)?\\s*" + units); Matcher matcher = pattern.matcher(val); if (!matcher.find()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index c08d13e9c48ca..8734877f6083a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -41,7 +41,6 @@ import com.google.common.annotations.VisibleForTesting; import java.util.Iterator; import java.util.Set; -import org.apache.hadoop.yarn.api.records.Resource; /** * Maintains a list of queues as well as scheduling parameters for each queue, @@ -330,7 +329,7 @@ void setChildResourceLimits(FSParentQueue parent, FSQueue child, !configuredQueues.get(FSQueueType.PARENT).contains(child.getName())) { // For ad hoc queues, set their max reource allocations based on // their parents' default child settings. - Resource maxChild = parent.getMaxChildQueueResource(); + ConfigurableResource maxChild = parent.getMaxChildQueueResource(); if (maxChild != null) { child.setMaxShare(maxChild); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index c8b9ad8bdefbc..67b46f9939867 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -235,21 +235,21 @@ public void testAllocationFileParsing() throws Exception { queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(2048, 10), - queueConf.getMaxResources("root.queueA")); + queueConf.getMaxResources("root.queueA").getResource()); assertEquals(Resources.createResource(5120, 110), - queueConf.getMaxResources("root.queueB")); - assertEquals(Resources.createResource(5120, 0), - queueConf.getMaxResources("root.queueC")); + queueConf.getMaxResources("root.queueB").getResource()); + assertEquals(Resources.createResource(4096, 100), + queueConf.getMaxResources("root.queueC").getResource()); assertEquals(Resources.createResource(4096, 100), - queueConf.getMaxResources("root.queueD")); + queueConf.getMaxResources("root.queueD").getResource()); assertEquals(Resources.createResource(4096, 100), - queueConf.getMaxResources("root.queueE")); + queueConf.getMaxResources("root.queueE").getResource()); assertEquals(Resources.createResource(4096, 100), - queueConf.getMaxResources("root.queueF")); + queueConf.getMaxResources("root.queueF").getResource()); assertEquals(Resources.createResource(4096, 100), - queueConf.getMaxResources("root.queueG")); + queueConf.getMaxResources("root.queueG").getResource()); assertEquals(Resources.createResource(4096, 100), - queueConf.getMaxResources("root.queueG.queueH")); + queueConf.getMaxResources("root.queueG.queueH").getResource()); assertEquals(Resources.createResource(1024, 0), queueConf.getMinResources("root.queueA")); @@ -279,9 +279,9 @@ public void testAllocationFileParsing() throws Exception { assertNull("Max child resources unexpectedly set for queue root.queueE", queueConf.getMaxChildResources("root.queueE")); assertEquals(Resources.createResource(2048, 64), - queueConf.getMaxChildResources("root.queueF")); + queueConf.getMaxChildResources("root.queueF").getResource()); assertEquals(Resources.createResource(2048, 64), - queueConf.getMaxChildResources("root.queueG")); + queueConf.getMaxChildResources("root.queueG").getResource()); assertNull("Max child resources unexpectedly set for " + "queue root.queueG.queueH", queueConf.getMaxChildResources("root.queueG.queueH")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java new file mode 100644 index 0000000000000..29143c69ae36b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TestConfigurableResource { + private final Resource clusterResource = Resources.createResource(2048, 2); + + @Test + public void testGetResourceWithPercentage() { + ConfigurableResource configurableResource = + new ConfigurableResource(new double[] { 0.5, 0.5 }); + assertEquals( + configurableResource.getResource(clusterResource).getMemorySize(), + 1024); + assertEquals( + configurableResource.getResource(clusterResource).getVirtualCores(), 1); + + assertNull(configurableResource.getResource()); + assertNull(configurableResource.getResource(null)); + } + + @Test + public void testGetResourceWithAbsolute() { + ConfigurableResource configurableResource = + new ConfigurableResource(Resources.createResource(3072, 3)); + assertEquals(configurableResource.getResource().getMemorySize(), 3072); + assertEquals(configurableResource.getResource().getVirtualCores(), 3); + + assertEquals( + configurableResource.getResource(clusterResource).getMemorySize(), + 3072); + assertEquals( + configurableResource.getResource(clusterResource).getVirtualCores(), + 3); + + assertEquals(configurableResource.getResource(null).getMemorySize(), 3072); + assertEquals(configurableResource.getResource(null).getVirtualCores(), 3); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java index b6b3f7ec6ca46..0bba35d90ca4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java @@ -81,7 +81,7 @@ public void testUpdateDemand() { String queueName = "root.queue1"; FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null); - schedulable.setMaxShare(maxResource); + schedulable.setMaxShare(new ConfigurableResource(maxResource)); assertEquals(schedulable.getMetrics().getMaxApps(), Integer.MAX_VALUE); assertEquals(schedulable.getMetrics().getSchedulingPolicy(), SchedulingPolicy.DEFAULT_POLICY.getName()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java index 3ae8f83215093..671b94be9ed12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.junit.Before; import org.junit.Test; @@ -42,6 +43,8 @@ public void setUp() throws Exception { AllocationConfiguration allocConf = new AllocationConfiguration(conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); + when(scheduler.getResourceCalculator()).thenReturn( + new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); notEmptyQueues = new HashSet(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index d5b1fcca272c1..6b2109d835644 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -4611,7 +4611,8 @@ public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); FSQueue queue2 = queueMgr.getLeafQueue("queue2", true); - queue2.setMaxShare(Resource.newInstance(1024, 1)); + queue2.setMaxShare( + new ConfigurableResource(Resource.newInstance(1024, 1))); ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); @@ -5209,6 +5210,8 @@ public void testUpdateDemand() throws IOException { scheduler.reinitialize(conf, resourceManager.getRMContext()); Resource maxResource = Resources.createResource(1024 * 8); + ConfigurableResource maxResourceConf = + new ConfigurableResource(maxResource); FSAppAttempt app1 = mock(FSAppAttempt.class); Mockito.when(app1.getDemand()).thenReturn(maxResource); @@ -5220,15 +5223,15 @@ public void testUpdateDemand() throws IOException { FSLeafQueue aQueue = new FSLeafQueue("root.queue1.a", scheduler, queue1); - aQueue.setMaxShare(maxResource); + aQueue.setMaxShare(maxResourceConf); aQueue.addApp(app1, true); FSLeafQueue bQueue = new FSLeafQueue("root.queue1.b", scheduler, queue1); - bQueue.setMaxShare(maxResource); + bQueue.setMaxShare(maxResourceConf); bQueue.addApp(app2, true); - queue1.setMaxShare(maxResource); + queue1.setMaxShare(maxResourceConf); queue1.addChildQueue(aQueue); queue1.addChildQueue(bQueue); @@ -5266,7 +5269,7 @@ public void testDumpState() throws IOException { FSLeafQueue child1 = scheduler.getQueueManager().getLeafQueue("parent.child1", false); Resource resource = Resource.newInstance(4 * GB, 4); - child1.setMaxShare(resource); + child1.setMaxShare(new ConfigurableResource(resource)); FSAppAttempt app = mock(FSAppAttempt.class); Mockito.when(app.getDemand()).thenReturn(resource); Mockito.when(app.getResourceUsage()).thenReturn(resource); @@ -5295,7 +5298,7 @@ public void testDumpState() throws IOException { childQueueString, child1.dumpState()); FSParentQueue parent = scheduler.getQueueManager().getParentQueue("parent", false); - parent.setMaxShare(resource); + parent.setMaxShare(new ConfigurableResource(resource)); parent.updateDemand(); String parentQueueString = "{Name: root.parent," diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index 8e7b666556399..999aaae2ca66e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -20,11 +20,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue; import static org.junit.Assert.assertEquals; -import java.io.File; - -import org.junit.Assert; - -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Test; @@ -32,23 +28,59 @@ public class TestFairSchedulerConfiguration { @Test public void testParseResourceConfigValue() throws Exception { assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("2 vcores, 1024 mb")); + parseResourceConfigValue("2 vcores, 1024 mb").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("1024 mb, 2 vcores").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("2vcores,1024mb").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("1024mb,2vcores").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("1024 mb, 2 vcores").getResource()); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 mb, 2 vcores")); + parseResourceConfigValue("1024 Mb, 2 vCores").getResource()); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("2vcores,1024mb")); + parseResourceConfigValue(" 1024 mb, 2 vcores ").getResource()); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024mb,2vcores")); + parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ").getResource()); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 mb, 2 vcores")); + parseResourceConfigValue(" 1024. mb, 2. vcores ").getResource()); + + Resource clusterResource = BuilderUtils.newResource(2048, 4); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("50% memory, 50% cpu"). + getResource(clusterResource)); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("50% Memory, 50% CpU"). + getResource(clusterResource)); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 Mb, 2 vCores")); + parseResourceConfigValue("50%").getResource(clusterResource)); + assertEquals(BuilderUtils.newResource(1024, 4), + parseResourceConfigValue("50% memory, 100% cpu"). + getResource(clusterResource)); + assertEquals(BuilderUtils.newResource(1024, 4), + parseResourceConfigValue(" 100% cpu, 50% memory"). + getResource(clusterResource)); + assertEquals(BuilderUtils.newResource(1024, 0), + parseResourceConfigValue("50% memory, 0% cpu"). + getResource(clusterResource)); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024 mb, 2 vcores ")); + parseResourceConfigValue("50 % memory, 50 % cpu"). + getResource(clusterResource)); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ")); + parseResourceConfigValue("50%memory,50%cpu"). + getResource(clusterResource)); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024. mb, 2. vcores ")); + parseResourceConfigValue(" 50 % memory, 50 % cpu "). + getResource(clusterResource)); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("50.% memory, 50.% cpu"). + getResource(clusterResource)); + + clusterResource = BuilderUtils.newResource(1024 * 10, 4); + assertEquals(BuilderUtils.newResource((int)(1024 * 10 * 0.109), 2), + parseResourceConfigValue("10.9% memory, 50.6% cpu"). + getResource(clusterResource)); } @Test(expected = AllocationConfigurationException.class) @@ -70,5 +102,24 @@ public void testOnlyCPU() throws Exception { public void testGibberish() throws Exception { parseResourceConfigValue("1o24vc0res"); } - + + @Test(expected = AllocationConfigurationException.class) + public void testNoUnitsPercentage() throws Exception { + parseResourceConfigValue("95%, 50% memory"); + } + + @Test(expected = AllocationConfigurationException.class) + public void testInvalidNumPercentage() throws Exception { + parseResourceConfigValue("95A% cpu, 50% memory"); + } + + @Test(expected = AllocationConfigurationException.class) + public void testCpuPercentageMemoryAbsolute() throws Exception { + parseResourceConfigValue("50% cpu, 1024 mb"); + } + + @Test(expected = AllocationConfigurationException.class) + public void testMemoryPercentageCpuAbsolute() throws Exception { + parseResourceConfigValue("50% memory, 2 vcores"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index f77df1ea3992a..964ecf560d496 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.util.ControlledClock; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.junit.Before; import org.junit.Test; @@ -55,7 +56,9 @@ public void setup() throws Exception { AllocationConfiguration allocConf = new AllocationConfiguration( conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); - + when(scheduler.getResourceCalculator()).thenReturn( + new DefaultResourceCalculator()); + queueManager = new QueueManager(scheduler); queueManager.initialize(conf); userMaxApps = allocConf.userMaxApps; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java index 05ab11c3150f7..eb2d402cd25df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Before; import org.junit.Test; @@ -50,6 +51,8 @@ public void setUp() throws Exception { when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); + when(scheduler.getResourceCalculator()).thenReturn( + new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); @@ -206,7 +209,7 @@ public void testCreateQueueWithChildDefaults() { queueManager.updateAllocationConfiguration(allocConf); queueManager.getQueue("root.test").setMaxChildQueueResource( - Resources.createResource(8192, 256)); + new ConfigurableResource(Resources.createResource(8192, 256))); FSQueue q1 = queueManager.createQueue("root.test.childC", FSQueueType.LEAF); assertNotNull("Leaf queue root.test.childC was not created", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java index 67d73409466ab..83b7e41112f59 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.junit.Assert; import org.junit.Test; @@ -43,6 +44,8 @@ public void testEmptyChildQueues() throws Exception { when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(1, 1)); + when(scheduler.getResourceCalculator()).thenReturn( + new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); QueueManager queueManager = new QueueManager(scheduler); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md index 748fda9be0b0b..17db7eea40779 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md @@ -85,9 +85,9 @@ The allocation file must be in XML format. The format contains five types of ele * **minResources**: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources. - * **maxResources**: maximum resources a queue is allowed, in the form "X mb, Y vcores". A queue will never be assigned a container that would put its aggregate usage over this limit. + * **maxResources**: maximum resources a queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). A queue will not be assigned a container that would put its aggregate usage over this limit. - * **maxChildResources**: maximum resources an ad hoc child queue is allowed, in the form "X mb, Y vcores". Any ad hoc queue that is a direct child of a queue with this property set will have it's maxResources property set accordingly. + * **maxChildResources**: maximum resources an ad hoc child queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit. * **maxRunningApps**: limit the number of apps from the queue to run at once From 977b6eab736d08639c0b3e624b041521181346dd Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Mon, 25 Sep 2017 10:35:23 -0700 Subject: [PATCH 2/4] YARN-2162 fixed style issues --- .../scheduler/fair/AllocationConfiguration.java | 2 -- .../scheduler/fair/TestConfigurableResource.java | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index f8aaf7de0346c..b8251cc224011 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -31,8 +31,6 @@ import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; -import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; -import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java index 29143c69ae36b..d32897f3977d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java @@ -25,13 +25,16 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +/** + * To test class {@link ConfigurableResource}. + */ public class TestConfigurableResource { private final Resource clusterResource = Resources.createResource(2048, 2); @Test public void testGetResourceWithPercentage() { ConfigurableResource configurableResource = - new ConfigurableResource(new double[] { 0.5, 0.5 }); + new ConfigurableResource(new double[] {0.5, 0.5}); assertEquals( configurableResource.getResource(clusterResource).getMemorySize(), 1024); From c68fe76adc0f48791a369a49119c88adf3d10c8d Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Thu, 28 Sep 2017 11:28:52 -0700 Subject: [PATCH 3/4] YARN-2162 --- .../scheduler/fair/AllocationFileLoaderService.java | 12 ++++++------ .../scheduler/fair/ConfigurableResource.java | 13 +++++++++++++ .../scheduler/fair/FairSchedulerConfiguration.java | 5 +++-- .../scheduler/fair/TestConfigurableResource.java | 7 +++++-- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index d58abd4f6b984..597af9411c019 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -610,14 +610,14 @@ private void loadQueue(String parentName, Element element, checkMinAndMaxResource(minQueueResources, maxQueueResources, queueName); } - private void checkMinAndMaxResource(Map minQueueResources, - Map maxQueueResources, String queueName) { + private void checkMinAndMaxResource(Map minResources, + Map maxResources, String queueName) { - if (maxQueueResources.containsKey(queueName) - && minQueueResources.containsKey(queueName)) { + ConfigurableResource maxConfigurableResource = maxResources.get(queueName); + Resource minResource = minResources.get(queueName); - Resource maxResource = maxQueueResources.get(queueName).getResource(); - Resource minResource = minQueueResources.get(queueName); + if (maxConfigurableResource != null && minResource != null) { + Resource maxResource = maxConfigurableResource.getResource(); // check whether max resource is bigger or equals to min resource when max // resource are absolute values diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java index fffcb96e5cd95..8de41be3c7bac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java @@ -43,6 +43,14 @@ public ConfigurableResource(Resource resource) { this.resource = resource; } + /** + * Get resource by multiplying the cluster resource and the percentage of + * each resource respectively. Return the absolute resource if either + * {@code percentages} or {@code clusterResource) is null. + * + * @param clusterResource the cluster resource + * @return resource + */ public Resource getResource(Resource clusterResource) { if (percentages != null && clusterResource != null) { long memory = (long) (clusterResource.getMemorySize() * percentages[0]); @@ -53,6 +61,11 @@ public Resource getResource(Resource clusterResource) { } } + /** + * Get the absolute resource. + * + * @return the absolute resource + */ public Resource getResource() { return resource; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 1abde8424c655..960299b70bfd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -329,7 +329,8 @@ private static double[] getResourcePercentage( private static double findPercentage(String val, String units) throws AllocationConfigurationException { - Pattern pattern = Pattern.compile("((\\d+)(\\.\\d*)?)\\s*%\\s*" + units); + final Pattern pattern = + Pattern.compile("((\\d+)(\\.\\d*)?)\\s*%\\s*" + units); Matcher matcher = pattern.matcher(val); if (!matcher.find()) { if (units.equals("")) { @@ -349,7 +350,7 @@ public long getUpdateInterval() { private static int findResource(String val, String units) throws AllocationConfigurationException { - Pattern pattern = Pattern.compile("(\\d+)(\\.\\d*)?\\s*" + units); + final Pattern pattern = Pattern.compile("(\\d+)(\\.\\d*)?\\s*" + units); Matcher matcher = pattern.matcher(val); if (!matcher.find()) { throw new AllocationConfigurationException("Missing resource: " + units); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java index d32897f3977d6..249d1f77ca6b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java @@ -41,8 +41,11 @@ public void testGetResourceWithPercentage() { assertEquals( configurableResource.getResource(clusterResource).getVirtualCores(), 1); - assertNull(configurableResource.getResource()); - assertNull(configurableResource.getResource(null)); + assertNull("The absolute resource should be null since object" + + " configurableResource is initialized with percentages", + configurableResource.getResource()); + assertNull("The absolute resource should be null since cluster resource" + + " is null", configurableResource.getResource(null)); } @Test From 65ce5f14dd2d8f8560e2b7389ee830c225a2df25 Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Thu, 5 Oct 2017 10:06:40 -0700 Subject: [PATCH 4/4] YARN-2162 --- .../scheduler/fair/AllocationConfiguration.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index b8251cc224011..7eb292f2f8712 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -292,7 +292,8 @@ Resource getMinResources(String queue) { * @param queue the target queue's name * @return the max allocation on this queue */ - @VisibleForTesting ConfigurableResource getMaxResources(String queue) { + @VisibleForTesting + ConfigurableResource getMaxResources(String queue) { ConfigurableResource maxQueueResource = maxQueueResources.get(queue); if (maxQueueResource == null) { maxQueueResource = queueMaxResourcesDefault;