YARN-7556. Fair scheduler configuration should allow resource types i…

…n the minResources and maxResources properties. (Daniel Templeton and Szilard Nemeth via Haibo Chen)
apache · Jul 6, 2018 · 9edc74f · 9edc74f
1 parent 39ad989
commit 9edc74f
Show file tree

Hide file tree

Showing 8 changed files with 385 additions and 70 deletions.
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -65,11 +65,6 @@
     <Class name="~org\.apache\.hadoop\.yarn\.server\.resourcemanager\.rmapp\.attempt\.RMAppAttemptImpl.*" />
     <Bug pattern="BC_UNCONFIRMED_CAST" />
   </Match>
-  <Match>
-    <Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
-    <Method name="getLocalityStatistics" />
-    <Bug pattern="EI_EXPOSE_REP" />
-  </Match>
   <Match>
     <Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
     <Method name="incNumAllocatedContainers"/>
@@ -118,6 +113,18 @@
     <Bug pattern="BC_UNCONFIRMED_CAST" />
   </Match>
 
+  <!-- Ignore exposed internal representations -->
+  <Match>
+    <Class name="org.apache.hadoop.yarn.api.records.Resource" />
+    <Method name="getResources" />
+    <Bug pattern="EI_EXPOSE_REP" />
+  </Match>
+  <Match>
+    <Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
+    <Method name="getLocalityStatistics" />
+    <Bug pattern="EI_EXPOSE_REP" />
+  </Match>
+
   <!-- Object cast is based on the event type -->
   <Match>
     <Class name="org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher" />

diff --git a/...adoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/...adoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.api.records.impl.LightWeightResource;
@@ -75,6 +76,18 @@ public abstract class Resource implements Comparable<Resource> {
   @Private
   public static final int VCORES_INDEX = 1;
 
+  /**
+   * Return a new {@link Resource} instance with all resource values
+   * initialized to {@code value}.
+   * @param value the value to use for all resources
+   * @return a new {@link Resource} instance
+   */
+  @Private
+  @Unstable
+  public static Resource newInstance(long value) {
+    return new LightWeightResource(value);
+  }
+
   @Public
   @Stable
   public static Resource newInstance(int memory, int vCores) {

diff --git a/...p-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java b/...p-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
@@ -18,9 +18,8 @@
 
 package org.apache.hadoop.yarn.api.records.impl;
 
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@@ -58,13 +57,29 @@
  *
  * @see Resource
  */
-@InterfaceAudience.Private
+@Private
 @Unstable
 public class LightWeightResource extends Resource {
 
   private ResourceInformation memoryResInfo;
   private ResourceInformation vcoresResInfo;
 
+  /**
+   * Create a new {@link LightWeightResource} instance with all resource values
+   * initialized to {@code value}.
+   * @param value the value to use for all resources
+   */
+  public LightWeightResource(long value) {
+    ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
+    initResourceInformations(value, value, types.length);
+
+    for (int i = 2; i < types.length; i++) {
+      resources[i] = new ResourceInformation();
+      ResourceInformation.copy(types[i], resources[i]);
+      resources[i].setValue(value);
+    }
+  }
+
   public LightWeightResource(long memory, int vcores) {
     int numberOfKnownResourceTypes = ResourceUtils
         .getNumberOfKnownResourceTypes();
@@ -91,7 +106,7 @@ public LightWeightResource(long memory, int vcores,
     }
   }
 
-  private void initResourceInformations(long memory, int vcores,
+  private void initResourceInformations(long memory, long vcores,
       int numberOfKnownResourceTypes) {
     this.memoryResInfo = newDefaultInformation(MEMORY_URI, MEMORY_MB.getUnits(),
         memory);

diff --git a/...va/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java b/...va/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
@@ -18,9 +18,13 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
+import java.util.Arrays;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 
 /**
  * A {@code ConfigurableResource} object represents an entity that is used to
@@ -33,29 +37,53 @@ public class ConfigurableResource {
   private final Resource resource;
   private final double[] percentages;
 
-  public ConfigurableResource(double[] percentages) {
+  ConfigurableResource() {
+    this(getOneHundredPercentArray());
+  }
+
+  ConfigurableResource(double[] percentages) {
     this.percentages = percentages.clone();
     this.resource = null;
   }
 
+  ConfigurableResource(long value) {
+    this(Resource.newInstance(value));
+  }
+
   public ConfigurableResource(Resource resource) {
     this.percentages = null;
     this.resource = resource;
   }
 
+  private static double[] getOneHundredPercentArray() {
+    double[] resourcePercentages =
+        new double[ResourceUtils.getNumberOfKnownResourceTypes()];
+    Arrays.fill(resourcePercentages, 1.0);
+
+    return resourcePercentages;
+  }
+
   /**
    * Get resource by multiplying the cluster resource and the percentage of
    * each resource respectively. Return the absolute resource if either
    * {@code percentages} or {@code clusterResource} is null.
    *
    * @param clusterResource the cluster resource
-   * @return resource
+   * @return resource the resulting resource
    */
   public Resource getResource(Resource clusterResource) {
     if (percentages != null && clusterResource != null) {
       long memory = (long) (clusterResource.getMemorySize() * percentages[0]);
       int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]);
-      return Resource.newInstance(memory, vcore);
+      Resource res = Resource.newInstance(memory, vcore);
+      ResourceInformation[] clusterInfo = clusterResource.getResources();
+
+      for (int i = 2; i < clusterInfo.length; i++) {
+        res.setResourceValue(i,
+            (long)(clusterInfo[i].getValue() * percentages[i]));
+      }
+
+      return res;
     } else {
       return resource;
     }
@@ -69,4 +97,39 @@ public Resource getResource(Resource clusterResource) {
   public Resource getResource() {
     return resource;
   }
+
+  /**
+   * Set the value of the wrapped resource if this object isn't setup to use
+   * percentages. If this object is set to use percentages, this method has
+   * no effect.
+   *
+   * @param name the name of the resource
+   * @param value the value
+   */
+  void setValue(String name, long value) {
+    if (resource != null) {
+      resource.setResourceValue(name, value);
+    }
+  }
+
+  /**
+   * Set the percentage of the resource if this object is setup to use
+   * percentages. If this object is set to use percentages, this method has
+   * no effect.
+   *
+   * @param name the name of the resource
+   * @param value the percentage
+   */
+  void setPercentage(String name, double value) {
+    if (percentages != null) {
+      Integer index = ResourceUtils.getResourceTypeIndex().get(name);
+
+      if (index != null) {
+        percentages[index] = value;
+      } else {
+        throw new ResourceNotFoundException("The requested resource, \""
+            + name + "\", could not be found.");
+      }
+    }
+  }
 }