Skip to content

Commit

Permalink
YARN-3521. Support return structured NodeLabel objects in REST API (S…
Browse files Browse the repository at this point in the history
…unil G via wangda)
  • Loading branch information
wangdatan committed May 13, 2015
1 parent d4f53fc commit 7f19e7a
Show file tree
Hide file tree
Showing 10 changed files with 427 additions and 134 deletions.
8 changes: 5 additions & 3 deletions hadoop-yarn-project/CHANGES.txt
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -233,6 +233,11 @@ Release 2.8.0 - UNRELEASED
YARN-3613. TestContainerManagerSecurity should init and start Yarn cluster in YARN-3613. TestContainerManagerSecurity should init and start Yarn cluster in
setup instead of individual methods. (nijel via kasha) setup instead of individual methods. (nijel via kasha)


YARN-3579. CommonNodeLabelsManager should support NodeLabel instead of string
label name when getting node-to-label/label-to-label mappings. (Sunil G via wangda)

YARN-3521. Support return structured NodeLabel objects in REST API (Sunil G via wangda)

OPTIMIZATIONS OPTIMIZATIONS


YARN-3339. TestDockerContainerExecutor should pull a single image and not YARN-3339. TestDockerContainerExecutor should pull a single image and not
Expand Down Expand Up @@ -420,9 +425,6 @@ Release 2.7.1 - UNRELEASED
YARN-3539. Updated timeline server documentation and marked REST APIs evolving. YARN-3539. Updated timeline server documentation and marked REST APIs evolving.
(Steve Loughran via zjshen) (Steve Loughran via zjshen)


YARN-3579. CommonNodeLabelsManager should support NodeLabel instead of string
label name when getting node-to-label/label-to-label mappings. (Sunil G via wangda)

OPTIMIZATIONS OPTIMIZATIONS


BUG FIXES BUG FIXES
Expand Down
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@


import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlRootElement;


@XmlRootElement(name = "labelsToNodesInfo") @XmlRootElement(name = "labelsToNodesInfo")
Expand All @@ -32,6 +33,7 @@ public class NodeIDsInfo {
/** /**
* Set doesn't support default no arg constructor which is req by JAXB * Set doesn't support default no arg constructor which is req by JAXB
*/ */
@XmlElement(name="nodes")
protected ArrayList<String> nodeIDsList = new ArrayList<String>(); protected ArrayList<String> nodeIDsList = new ArrayList<String>();


public NodeIDsInfo() { public NodeIDsInfo() {
Expand Down
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueACL;
Expand Down Expand Up @@ -135,8 +136,11 @@
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntry;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
Expand Down Expand Up @@ -796,18 +800,18 @@ public Response updateAppState(AppState targetState,
@GET @GET
@Path("/get-node-to-labels") @Path("/get-node-to-labels")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodeToLabelsInfo getNodeToLabels(@Context HttpServletRequest hsr) public NodeToLabelsInfo getNodeToLabels(@Context HttpServletRequest hsr)
throws IOException { throws IOException {
init(); init();


NodeToLabelsInfo ntl = new NodeToLabelsInfo(); NodeToLabelsInfo ntl = new NodeToLabelsInfo();
HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels(); HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels();
Map<NodeId, Set<String>> nodeIdToLabels = Map<NodeId, Set<String>> nodeIdToLabels = rm.getRMContext()
rm.getRMContext().getNodeLabelManager().getNodeLabels(); .getNodeLabelManager().getNodeLabels();

for (Map.Entry<NodeId, Set<String>> nitle : nodeIdToLabels.entrySet()) { for (Map.Entry<NodeId, Set<String>> nitle : nodeIdToLabels.entrySet()) {
ntlMap.put(nitle.getKey().toString(), ntlMap.put(nitle.getKey().toString(),
new NodeLabelsInfo(nitle.getValue())); new NodeLabelsInfo(nitle.getValue()));
} }


return ntl; return ntl;
Expand All @@ -821,7 +825,7 @@ public LabelsToNodesInfo getLabelsToNodes(
init(); init();


LabelsToNodesInfo lts = new LabelsToNodesInfo(); LabelsToNodesInfo lts = new LabelsToNodesInfo();
Map<String, NodeIDsInfo> ltsMap = lts.getLabelsToNodes(); Map<NodeLabelInfo, NodeIDsInfo> ltsMap = lts.getLabelsToNodes();
Map<String, Set<NodeId>> labelsToNodeId = null; Map<String, Set<NodeId>> labelsToNodeId = null;
if (labels == null || labels.size() == 0) { if (labels == null || labels.size() == 0) {
labelsToNodeId = labelsToNodeId =
Expand All @@ -836,24 +840,24 @@ public LabelsToNodesInfo getLabelsToNodes(
for (NodeId nodeId : entry.getValue()) { for (NodeId nodeId : entry.getValue()) {
nodeIdStrList.add(nodeId.toString()); nodeIdStrList.add(nodeId.toString());
} }
ltsMap.put(entry.getKey(), new NodeIDsInfo(nodeIdStrList)); ltsMap.put(new NodeLabelInfo(entry.getKey()), new NodeIDsInfo(
nodeIdStrList));
} }
return lts; return lts;
} }


@POST @POST
@Path("/replace-node-to-labels") @Path("/replace-node-to-labels")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public Response replaceLabelsOnNodes(final NodeToLabelsInfo newNodeToLabels, public Response replaceLabelsOnNodes(final NodeToLabelsEntryList newNodeToLabels,
@Context HttpServletRequest hsr) throws IOException { @Context HttpServletRequest hsr) throws IOException {
Map<NodeId, Set<String>> nodeIdToLabels = Map<NodeId, Set<String>> nodeIdToLabels =
new HashMap<NodeId, Set<String>>(); new HashMap<NodeId, Set<String>>();


for (Map.Entry<String, NodeLabelsInfo> nitle : newNodeToLabels for (NodeToLabelsEntry nitle : newNodeToLabels.getNodeToLabels()) {
.getNodeToLabels().entrySet()) {
nodeIdToLabels.put( nodeIdToLabels.put(
ConverterUtils.toNodeIdWithDefaultPort(nitle.getKey()), ConverterUtils.toNodeIdWithDefaultPort(nitle.getNodeId()),
new HashSet<String>(nitle.getValue().getNodeLabels())); new HashSet<String>(nitle.getNodeLabels()));
} }


return replaceLabelsOnNode(nodeIdToLabels, hsr, "/replace-node-to-labels"); return replaceLabelsOnNode(nodeIdToLabels, hsr, "/replace-node-to-labels");
Expand All @@ -862,16 +866,18 @@ public Response replaceLabelsOnNodes(final NodeToLabelsInfo newNodeToLabels,
@POST @POST
@Path("/nodes/{nodeId}/replace-labels") @Path("/nodes/{nodeId}/replace-labels")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public Response replaceLabelsOnNode(NodeLabelsInfo newNodeLabelsInfo, public Response replaceLabelsOnNode(
@QueryParam("labels") Set<String> newNodeLabelsName,
@Context HttpServletRequest hsr, @PathParam("nodeId") String nodeId) @Context HttpServletRequest hsr, @PathParam("nodeId") String nodeId)
throws Exception { throws Exception {
NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId); NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId);
Map<NodeId, Set<String>> newLabelsForNode = Map<NodeId, Set<String>> newLabelsForNode =
new HashMap<NodeId, Set<String>>(); new HashMap<NodeId, Set<String>>();
newLabelsForNode.put(nid, newLabelsForNode.put(nid,
new HashSet<String>(newNodeLabelsInfo.getNodeLabels())); new HashSet<String>(newNodeLabelsName));


return replaceLabelsOnNode(newLabelsForNode, hsr, "/nodes/nodeid/replace-labels"); return replaceLabelsOnNode(newLabelsForNode, hsr,
"/nodes/nodeid/replace-labels");
} }


private Response replaceLabelsOnNode( private Response replaceLabelsOnNode(
Expand Down Expand Up @@ -909,9 +915,9 @@ public NodeLabelsInfo getClusterNodeLabels(@Context HttpServletRequest hsr)
throws IOException { throws IOException {
init(); init();


NodeLabelsInfo ret = List<NodeLabel> nodeLabels = rm.getRMContext().getNodeLabelManager()
new NodeLabelsInfo(rm.getRMContext().getNodeLabelManager() .getClusterNodeLabels();
.getClusterNodeLabelNames()); NodeLabelsInfo ret = new NodeLabelsInfo(nodeLabels);


return ret; return ret;
} }
Expand All @@ -937,8 +943,7 @@ public Response addToClusterNodeLabels(final NodeLabelsInfo newNodeLabels,
} }


rm.getRMContext().getNodeLabelManager() rm.getRMContext().getNodeLabelManager()
.addToCluserNodeLabelsWithDefaultExclusivity(new HashSet<String>( .addToCluserNodeLabels(newNodeLabels.getNodeLabels());
newNodeLabels.getNodeLabels()));


return Response.status(Status.OK).build(); return Response.status(Status.OK).build();


Expand All @@ -947,29 +952,29 @@ public Response addToClusterNodeLabels(final NodeLabelsInfo newNodeLabels,
@POST @POST
@Path("/remove-node-labels") @Path("/remove-node-labels")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public Response removeFromCluserNodeLabels(final NodeLabelsInfo oldNodeLabels, public Response removeFromCluserNodeLabels(
@Context HttpServletRequest hsr) @QueryParam("labels") Set<String> oldNodeLabels,
throws Exception { @Context HttpServletRequest hsr) throws Exception {
init(); init();


UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) { if (callerUGI == null) {
String msg = "Unable to obtain user name, user not authenticated for" String msg = "Unable to obtain user name, user not authenticated for"
+ " post to .../remove-node-labels"; + " post to .../remove-node-labels";
throw new AuthorizationException(msg); throw new AuthorizationException(msg);
} }
if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) {
String msg = "User " + callerUGI.getShortUserName() + " not authorized" String msg = "User " + callerUGI.getShortUserName() + " not authorized"
+ " for post to .../remove-node-labels "; + " for post to .../remove-node-labels ";
throw new AuthorizationException(msg); throw new AuthorizationException(msg);
} }

rm.getRMContext().getNodeLabelManager()
.removeFromClusterNodeLabels(new HashSet<String>(
oldNodeLabels.getNodeLabels()));

return Response.status(Status.OK).build();


rm.getRMContext()
.getNodeLabelManager()
.removeFromClusterNodeLabels(
new HashSet<String>(oldNodeLabels));

return Response.status(Status.OK).build();
} }


@GET @GET
Expand Down
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -31,13 +31,13 @@
@XmlAccessorType(XmlAccessType.FIELD) @XmlAccessorType(XmlAccessType.FIELD)
public class LabelsToNodesInfo { public class LabelsToNodesInfo {


protected Map<String, NodeIDsInfo> labelsToNodes = protected Map<NodeLabelInfo, NodeIDsInfo> labelsToNodes =
new HashMap<String, NodeIDsInfo>(); new HashMap<NodeLabelInfo, NodeIDsInfo>();


public LabelsToNodesInfo() { public LabelsToNodesInfo() {
} // JAXB needs this } // JAXB needs this


public Map<String, NodeIDsInfo> getLabelsToNodes() { public Map<NodeLabelInfo, NodeIDsInfo> getLabelsToNodes() {
return labelsToNodes; return labelsToNodes;
} }
} }
Original file line number Original file line Diff line number Diff line change
@@ -0,0 +1,86 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;

import org.apache.hadoop.yarn.api.records.NodeLabel;

@XmlRootElement(name = "nodeLabelInfo")
@XmlAccessorType(XmlAccessType.FIELD)
public class NodeLabelInfo {

private String name;
private boolean exclusivity;

public NodeLabelInfo() {
// JAXB needs this
}

public NodeLabelInfo(String name) {
this.name = name;
this.exclusivity = true;
}

public NodeLabelInfo(String name, boolean exclusivity) {
this.name = name;
this.exclusivity = exclusivity;
}

public NodeLabelInfo(NodeLabel label) {
this.name = label.getName();
this.exclusivity = label.isExclusive();
}

public String getName() {
return name;
}

public boolean getExclusivity() {
return exclusivity;
}

@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
NodeLabelInfo other = (NodeLabelInfo) obj;
if (!getName().equals(other.getName())) {
return false;
}
if (getExclusivity() != other.getExclusivity()) {
return false;
}
return true;
}

@Override
public int hashCode() {
return (getName().hashCode() << 16) + (getExclusivity() ? 1 : 0);
}
}
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -22,31 +22,63 @@


import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlRootElement;


import org.apache.hadoop.yarn.api.records.NodeLabel;

@XmlRootElement(name = "nodeLabelsInfo") @XmlRootElement(name = "nodeLabelsInfo")
@XmlAccessorType(XmlAccessType.FIELD) @XmlAccessorType(XmlAccessType.FIELD)
public class NodeLabelsInfo { public class NodeLabelsInfo {


protected ArrayList<String> nodeLabels = new ArrayList<String>(); @XmlElement(name = "nodeLabelInfo")
private ArrayList<NodeLabelInfo> nodeLabelsInfo =
new ArrayList<NodeLabelInfo>();


public NodeLabelsInfo() { public NodeLabelsInfo() {
} // JAXB needs this // JAXB needs this

public NodeLabelsInfo(ArrayList<String> nodeLabels) {
this.nodeLabels = nodeLabels;
} }


public NodeLabelsInfo(Set<String> nodeLabelsSet) { public NodeLabelsInfo(ArrayList<NodeLabelInfo> nodeLabels) {
this.nodeLabels = new ArrayList<String>(nodeLabelsSet); this.nodeLabelsInfo = nodeLabels;
}

public NodeLabelsInfo(List<NodeLabel> nodeLabels) {
this.nodeLabelsInfo = new ArrayList<NodeLabelInfo>();
for (NodeLabel label : nodeLabels) {
this.nodeLabelsInfo.add(new NodeLabelInfo(label));
}
} }


public ArrayList<String> getNodeLabels() { public NodeLabelsInfo(Set<String> nodeLabelsName) {
this.nodeLabelsInfo = new ArrayList<NodeLabelInfo>();
for (String labelName : nodeLabelsName) {
this.nodeLabelsInfo.add(new NodeLabelInfo(labelName));
}
}

public ArrayList<NodeLabelInfo> getNodeLabelsInfo() {
return nodeLabelsInfo;
}

public Set<NodeLabel> getNodeLabels() {
Set<NodeLabel> nodeLabels = new HashSet<NodeLabel>();
for (NodeLabelInfo label : nodeLabelsInfo) {
nodeLabels.add(NodeLabel.newInstance(label.getName(),
label.getExclusivity()));
}
return nodeLabels; return nodeLabels;
} }


public void setNodeLabels(ArrayList<String> nodeLabels) { public List<String> getNodeLabelsName() {
this.nodeLabels = nodeLabels; ArrayList<String> nodeLabelsName = new ArrayList<String>();
for (NodeLabelInfo label : nodeLabelsInfo) {
nodeLabelsName.add(label.getName());
}
return nodeLabelsName;
}

public void setNodeLabelsInfo(ArrayList<NodeLabelInfo> nodeLabelInfo) {
this.nodeLabelsInfo = nodeLabelInfo;
} }

} }
Loading

0 comments on commit 7f19e7a

Please sign in to comment.