YARN-5587. Add support for resource profiles. (vvasudev via asuresh)

hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml

@@ -153,6 +153,10 @@
     <Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceRetentionSet$LRUComparator" />
     <Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
   </Match>
+  <Match>
+    <Class name="org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl$ProfileCapabilityComparator" />
+    <Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
+  </Match>
   <Match>
     <Class name="org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl" />
     <Field name="builder" />

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java

@@ -204,4 +204,12 @@ public abstract void setContainersFromPreviousAttempts(
   @Unstable
   public abstract void setSchedulerResourceTypes(
       EnumSet<SchedulerResourceTypes> types);
+
+  @Public
+  @Unstable
+  public abstract Map<String, Resource> getResourceProfiles();
+
+  @Private
+  @Unstable
+  public abstract void setResourceProfiles(Map<String, Resource> profiles);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java

@@ -18,41 +18,93 @@
 
 package org.apache.hadoop.yarn.api.records;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.yarn.util.Records;
 
+import java.util.Map;
+
 /**
  * Class to capture capability requirements when using resource profiles. The
  * ProfileCapability is meant to be used as part of the ResourceRequest. A
  * profile capability has two pieces - the resource profile name and the
  * overrides. The resource profile specifies the name of the resource profile
  * to be used and the capability override is the overrides desired on specific
- * resource types. For example, you could use the "minimum" profile and set the
- * memory in the capability override to 4096M. This implies that you wish for
- * the resources specified in the "minimum" profile but with 4096M memory. The
- * conversion from the ProfileCapability to the Resource class with the actual
- * resource requirements will be done by the ResourceManager, which has the
- * actual profile to Resource mapping.
+ * resource types.
+ *
+ * For example, if you have a resource profile "small" that maps to
+ * {@literal <4096M, 2 cores, 1 gpu>} and you set the capability override to
+ * {@literal <8192M, 0 cores, 0 gpu>}, then the actual resource allocation on
+ * the ResourceManager will be {@literal <8192M, 2 cores, 1 gpu>}.
+ *
+ * Note that the conversion from the ProfileCapability to the Resource class
+ * with the actual resource requirements will be done by the ResourceManager,
+ * which has the actual profile to Resource mapping.
+ *
  */
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
 public abstract class ProfileCapability {
 
+  public static final String DEFAULT_PROFILE = "default";
+
+  public static ProfileCapability newInstance(Resource override) {
+    return newInstance(DEFAULT_PROFILE, override);
+  }
+
+  public static ProfileCapability newInstance(String profile) {
+    Preconditions
+        .checkArgument(profile != null, "The profile name cannot be null");
+    ProfileCapability obj = Records.newRecord(ProfileCapability.class);
+    obj.setProfileName(profile);
+    obj.setProfileCapabilityOverride(Resource.newInstance(0, 0));
+    return obj;
+  }
+
   public static ProfileCapability newInstance(String profile,
       Resource override) {
+    Preconditions
+        .checkArgument(profile != null, "The profile name cannot be null");
     ProfileCapability obj = Records.newRecord(ProfileCapability.class);
     obj.setProfileName(profile);
     obj.setProfileCapabilityOverride(override);
     return obj;
   }
 
+  /**
+   * Get the profile name.
+   * @return the profile name
+   */
   public abstract String getProfileName();
 
+  /**
+   * Get the profile capability override.
+   * @return Resource object containing the override.
+   */
   public abstract Resource getProfileCapabilityOverride();
 
+  /**
+   * Set the resource profile name.
+   * @param profileName the resource profile name
+   */
   public abstract void setProfileName(String profileName);
 
+  /**
+   * Set the capability override to override specific resource types on the
+   * resource profile.
+   *
+   * For example, if you have a resource profile "small" that maps to
+   * {@literal <4096M, 2 cores, 1 gpu>} and you set the capability override to
+   * {@literal <8192M, 0 cores, 0 gpu>}, then the actual resource allocation on
+   * the ResourceManager will be {@literal <8192M, 2 cores, 1 gpu>}.
+   *
+   * Note that the conversion from the ProfileCapability to the Resource class
+   * with the actual resource requirements will be done by the ResourceManager,
+   * which has the actual profile to Resource mapping.
+   *
+   * @param r Resource object containing the capability override
+   */
   public abstract void setProfileCapabilityOverride(Resource r);
 
   @Override
@@ -85,4 +137,34 @@ public String toString() {
     return "{ profile: " + this.getProfileName() + ", capabilityOverride: "
         + this.getProfileCapabilityOverride() + " }";
   }
+
+  /**
+   * Get a representation of the capability as a Resource object.
+   * @param capability the capability we wish to convert
+   * @param resourceProfilesMap map of profile name to Resource object
+   * @return Resource object representing the capability
+   */
+  public static Resource toResource(ProfileCapability capability,
+      Map<String, Resource> resourceProfilesMap) {
+    Preconditions
+        .checkArgument(capability != null, "Capability cannot be null");
+    Preconditions.checkArgument(resourceProfilesMap != null,
+        "Resource profiles map cannot be null");
+    Resource resource = Resource.newInstance(0, 0);
+
+    if (resourceProfilesMap.containsKey(capability.getProfileName())) {
+      resource = Resource
+          .newInstance(resourceProfilesMap.get(capability.getProfileName()));
+    }
+
+    if(capability.getProfileCapabilityOverride()!= null) {
+      for (Map.Entry<String, ResourceInformation> entry : capability
+          .getProfileCapabilityOverride().getResources().entrySet()) {
+        if (entry.getValue() != null && entry.getValue().getValue() != 0) {
+          resource.setResourceInformation(entry.getKey(), entry.getValue());
+        }
+      }
+    }
+    return resource;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java

@@ -19,7 +19,9 @@
 package org.apache.hadoop.yarn.api.records;
 
 import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
@@ -101,6 +103,18 @@ public static Resource newInstance(long memory, int vCores) {
     return new SimpleResource(memory, vCores);
   }
 
+  @InterfaceAudience.Private
+  @InterfaceStability.Unstable
+  public static Resource newInstance(Resource resource) {
+    Resource ret = Resource.newInstance(0, 0);
+    for (Map.Entry<String, ResourceInformation> entry : resource.getResources()
+        .entrySet()) {
+      ret.setResourceInformation(entry.getKey(),
+          ResourceInformation.newInstance(entry.getValue()));
+    }
+    return ret;
+  }
+
   /**
    * This method is DEPRECATED:
    * Use {@link Resource#getMemorySize()} instead

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java

@@ -31,6 +31,8 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
   private String units;
   private ResourceTypes resourceType;
   private Long value;
+  private Long minimumAllocation;
+  private Long maximumAllocation;
 
   private static final String MEMORY_URI = "memory-mb";
   private static final String VCORES_URI = "vcores";
@@ -117,6 +119,42 @@ public void setValue(Long rValue) {
     this.value = rValue;
   }
 
+  /**
+   * Get the minimum allocation for the resource.
+   *
+   * @return the minimum allocation for the resource
+   */
+  public Long getMinimumAllocation() {
+    return minimumAllocation;
+  }
+
+  /**
+   * Set the minimum allocation for the resource.
+   *
+   * @param minimumAllocation the minimum allocation for the resource
+   */
+  public void setMinimumAllocation(Long minimumAllocation) {
+    this.minimumAllocation = minimumAllocation;
+  }
+
+  /**
+   * Get the maximum allocation for the resource.
+   *
+   * @return the maximum allocation for the resource
+   */
+  public Long getMaximumAllocation() {
+    return maximumAllocation;
+  }
+
+  /**
+   * Set the maximum allocation for the resource.
+   *
+   * @param maximumAllocation the maximum allocation for the resource
+   */
+  public void setMaximumAllocation(Long maximumAllocation) {
+    this.maximumAllocation = maximumAllocation;
+  }
+
   /**
    * Create a new instance of ResourceInformation from another object.
    *
@@ -129,33 +167,41 @@ public static ResourceInformation newInstance(ResourceInformation other) {
     ret.setResourceType(other.getResourceType());
     ret.setUnits(other.getUnits());
     ret.setValue(other.getValue());
+    ret.setMinimumAllocation(other.getMinimumAllocation());
+    ret.setMaximumAllocation(other.getMaximumAllocation());
     return ret;
   }
 
   public static ResourceInformation newInstance(String name, String units,
-      Long value, ResourceTypes type) {
+      Long value, ResourceTypes type, Long minimumAllocation,
+      Long maximumAllocation) {
     ResourceInformation ret = new ResourceInformation();
     ret.setName(name);
     ret.setResourceType(type);
     ret.setUnits(units);
     ret.setValue(value);
+    ret.setMinimumAllocation(minimumAllocation);
+    ret.setMaximumAllocation(maximumAllocation);
     return ret;
   }
 
   public static ResourceInformation newInstance(String name, String units,
       Long value) {
     return ResourceInformation
-        .newInstance(name, units, value, ResourceTypes.COUNTABLE);
+        .newInstance(name, units, value, ResourceTypes.COUNTABLE, 0L,
+            Long.MAX_VALUE);
   }
 
   public static ResourceInformation newInstance(String name, String units) {
     return ResourceInformation
-        .newInstance(name, units, 0L, ResourceTypes.COUNTABLE);
+        .newInstance(name, units, 0L, ResourceTypes.COUNTABLE, 0L,
+            Long.MAX_VALUE);
   }
 
   public static ResourceInformation newInstance(String name, Long value) {
     return ResourceInformation
-        .newInstance(name, "", value, ResourceTypes.COUNTABLE);
+        .newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,
+            Long.MAX_VALUE);
   }
 
   public static ResourceInformation newInstance(String name) {
@@ -165,7 +211,8 @@ public static ResourceInformation newInstance(String name) {
   @Override
   public String toString() {
     return "name: " + this.name + ", units: " + this.units + ", type: "
-        + resourceType + ", value: " + value;
+        + resourceType + ", value: " + value + ", minimum allocation: "
+        + minimumAllocation + ", maximum allocation: " + maximumAllocation;
   }
 
   public String getShorthandRepresentation() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java

@@ -98,7 +98,22 @@ public static ResourceRequest newInstance(Priority priority, String hostName,
         .resourceName(hostName).capability(capability)
         .numContainers(numContainers).relaxLocality(relaxLocality)
         .nodeLabelExpression(labelExpression)
-        .executionTypeRequest(executionTypeRequest).build();
+        .executionTypeRequest(executionTypeRequest).profileCapability(null)
+        .build();
+  }
+
+  @Public
+  @Evolving
+  public static ResourceRequest newInstance(Priority priority, String hostName,
+      Resource capability, int numContainers, boolean relaxLocality,
+      String labelExpression, ExecutionTypeRequest executionTypeRequest,
+      ProfileCapability profile) {
+    return ResourceRequest.newBuilder().priority(priority)
+        .resourceName(hostName).capability(capability)
+        .numContainers(numContainers).relaxLocality(relaxLocality)
+        .nodeLabelExpression(labelExpression)
+        .executionTypeRequest(executionTypeRequest).profileCapability(profile)
+        .build();
   }
 
   @Public
@@ -124,6 +139,7 @@ private ResourceRequestBuilder() {
       resourceRequest.setRelaxLocality(true);
       resourceRequest.setExecutionTypeRequest(
           ExecutionTypeRequest.newInstance());
+      resourceRequest.setProfileCapability(null);
     }
 
     /**
@@ -237,6 +253,21 @@ public ResourceRequestBuilder allocationRequestId(
       return this;
     }
 
+    /**
+     * Set the <code>resourceProfile</code> of the request.
+     * @see ResourceRequest#setProfileCapability(ProfileCapability)
+     * @param profileCapability
+     *          <code>profileCapability</code> of the request
+     * @return {@link ResourceRequestBuilder}
+     */
+    @Public
+    @Evolving
+    public ResourceRequestBuilder profileCapability(
+        ProfileCapability profileCapability) {
+      resourceRequest.setProfileCapability(profileCapability);
+      return this;
+    }
+
     /**
      * Return generated {@link ResourceRequest} object.
      * @return {@link ResourceRequest}
@@ -454,6 +485,14 @@ public ExecutionTypeRequest getExecutionTypeRequest() {
   @Evolving
   public abstract void setNodeLabelExpression(String nodelabelExpression);
 
+  @Public
+  @Evolving
+  public abstract ProfileCapability getProfileCapability();
+
+  @Public
+  @Evolving
+  public abstract void setProfileCapability(ProfileCapability p);
+
   /**
    * Get the optional <em>ID</em> corresponding to this allocation request. This
    * ID is an identifier for different {@code ResourceRequest}s from the <b>same
@@ -529,12 +568,14 @@ public int hashCode() {
     Resource capability = getCapability();
     String hostName = getResourceName();
     Priority priority = getPriority();
+    ProfileCapability profile = getProfileCapability();
     result =
         prime * result + ((capability == null) ? 0 : capability.hashCode());
     result = prime * result + ((hostName == null) ? 0 : hostName.hashCode());
     result = prime * result + getNumContainers();
     result = prime * result + ((priority == null) ? 0 : priority.hashCode());
     result = prime * result + Long.valueOf(getAllocationRequestId()).hashCode();
+    result = prime * result + ((profile == null) ? 0 : profile.hashCode());
     return result;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml

@@ -144,6 +144,7 @@
             <exclude>src/test/resources/application_1440536969523_0001.har/part-0</exclude>
             <exclude>src/test/resources/application_1440536969523_0001.har/_masterindex</exclude>
             <exclude>src/test/resources/application_1440536969523_0001.har/_SUCCESS</exclude>
+            <exclude>src/test/resources/resource-profiles.json</exclude>
           </excludes>
         </configuration>
       </plugin>