Skip to content

Commit

Permalink
YARN-3647. RMWebServices api's should use updated api from CommonNode…
Browse files Browse the repository at this point in the history 
…LabelsManager to get NodeLabel object. (Sunil G via wangda)
  • Loading branch information
@wangdatan
wangdatan committed May 27, 2015
1 parent c46d4ba commit ec0a852
Show file tree
Hide file tree
Showing 5 changed files with 62 additions and 37 deletions.
3 changes: 3 additions & 0 deletions hadoop-yarn-project/CHANGES.txt
View file
Expand Up @@ -265,6 +265,9 @@ Release 2.8.0 - UNRELEASED
YARN-3594. WintuilsProcessStubExecutor.startStreamReader leaks streams.
(Lars Francke via junping_du)

YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager
to get NodeLabel object. (Sunil G via wangda)

OPTIMIZATIONS

YARN-3339. TestDockerContainerExecutor should pull a single image and not
Expand Down
17 changes: 11 additions & 6 deletions ...-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
View file
Expand Up @@ -993,13 +993,18 @@ protected Set<String> getLabelsByNode(NodeId nodeId, Map<String, Host> map) {
}
}

private Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) {
Set<String> labels = getLabelsByNode(nodeId, nodeCollections);
if (labels.isEmpty()) {
return EMPTY_NODELABEL_SET;
public Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) {
try {
readLock.lock();
Set<String> labels = getLabelsByNode(nodeId, nodeCollections);
if (labels.isEmpty()) {
return EMPTY_NODELABEL_SET;
}
Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels);
return nodeLabels;
} finally {
readLock.unlock();
}
Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels);
return nodeLabels;
}

private Set<NodeLabel> createNodeLabelFromLabelNames(Set<String> labels) {
Expand Down
1 change: 1 addition & 0 deletions ...in/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
View file
Expand Up @@ -34,6 +34,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.RMNodeLabel;
Expand Down
27 changes: 13 additions & 14 deletions ...ger/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View file
Expand Up @@ -810,12 +810,12 @@ public NodeToLabelsInfo getNodeToLabels(@Context HttpServletRequest hsr)

NodeToLabelsInfo ntl = new NodeToLabelsInfo();
HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels();
Map<NodeId, Set<String>> nodeIdToLabels = rm.getRMContext()
.getNodeLabelManager().getNodeLabels();
Map<NodeId, Set<NodeLabel>> nodeIdToLabels = rm.getRMContext()
.getNodeLabelManager().getNodeLabelsInfo();

for (Map.Entry<NodeId, Set<String>> nitle : nodeIdToLabels.entrySet()) {
ntlMap.put(nitle.getKey().toString(),
new NodeLabelsInfo(nitle.getValue()));
for (Map.Entry<NodeId, Set<NodeLabel>> nitle : nodeIdToLabels.entrySet()) {
List<NodeLabel> labels = new ArrayList<NodeLabel>(nitle.getValue());
ntlMap.put(nitle.getKey().toString(), new NodeLabelsInfo(labels));
}

return ntl;
Expand All @@ -830,16 +830,16 @@ public LabelsToNodesInfo getLabelsToNodes(

LabelsToNodesInfo lts = new LabelsToNodesInfo();
Map<NodeLabelInfo, NodeIDsInfo> ltsMap = lts.getLabelsToNodes();
Map<String, Set<NodeId>> labelsToNodeId = null;
Map<NodeLabel, Set<NodeId>> labelsToNodeId = null;
if (labels == null || labels.size() == 0) {
labelsToNodeId =
rm.getRMContext().getNodeLabelManager().getLabelsToNodes();
rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes();
} else {
labelsToNodeId =
rm.getRMContext().getNodeLabelManager().getLabelsToNodes(labels);
rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes(labels);
}

for (Entry<String, Set<NodeId>> entry : labelsToNodeId.entrySet()) {
for (Entry<NodeLabel, Set<NodeId>> entry : labelsToNodeId.entrySet()) {
List<String> nodeIdStrList = new ArrayList<String>();
for (NodeId nodeId : entry.getValue()) {
nodeIdStrList.add(nodeId.toString());
Expand Down Expand Up @@ -985,14 +985,13 @@ public Response removeFromCluserNodeLabels(
@Path("/nodes/{nodeId}/get-labels")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr,
@PathParam("nodeId") String nodeId)
throws IOException {
@PathParam("nodeId") String nodeId) throws IOException {
init();

NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId);
return new NodeLabelsInfo(
rm.getRMContext().getNodeLabelManager().getLabelsOnNode(nid));

List<NodeLabel> labels = new ArrayList<NodeLabel>(rm.getRMContext()
.getNodeLabelManager().getLabelsInfoByNode(nid));
return new NodeLabelsInfo(labels);
}

protected Response killApp(RMApp app, UserGroupInformation callerUGI,
Expand Down
51 changes: 34 additions & 17 deletions ...ava/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
View file
Expand Up @@ -133,12 +133,15 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName());
assertEquals(1, nlsifo.getNodeLabels().size());
for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
assertEquals("a", nl.getName());
assertTrue(nl.getExclusivity());
}

// Add another
nlsifo = new NodeLabelsInfo();
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b"));
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b", false));
response =
r.path("ws").path("v1").path("cluster")
.path("add-node-labels").queryParam("user.name", userName)
Expand All @@ -154,6 +157,12 @@ public void testNodeLabels() throws JSONException, Exception {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals(2, nlsifo.getNodeLabels().size());
// Verify exclusivity for 'y' as false
for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
if (nl.getName().equals("b")) {
assertFalse(nl.getExclusivity());
}
}

// Add labels to a node
MultivaluedMapImpl params = new MultivaluedMapImpl();
Expand Down Expand Up @@ -202,7 +211,8 @@ public void testNodeLabels() throws JSONException, Exception {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class);
assertEquals(2, ltni.getLabelsToNodes().size());
NodeIDsInfo nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("b"));
NodeIDsInfo nodes = ltni.getLabelsToNodes().get(
new NodeLabelInfo("b", false));
assertTrue(nodes.getNodeIDs().contains("nid2:0"));
assertTrue(nodes.getNodeIDs().contains("nid1:0"));
nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("a"));
Expand Down Expand Up @@ -231,7 +241,7 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("a"));
assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));


// Replace
Expand All @@ -255,7 +265,8 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("b"));
assertTrue(nlsifo.getNodeLabelsInfo().contains(
new NodeLabelInfo("b", false)));

// Replace labels using node-to-labels
NodeToLabelsEntryList ntli = new NodeToLabelsEntryList();
Expand All @@ -281,7 +292,7 @@ public void testNodeLabels() throws JSONException, Exception {
NodeToLabelsInfo ntlinfo = response.getEntity(NodeToLabelsInfo.class);
NodeLabelsInfo nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size());
assertTrue(nlinfo.getNodeLabelsName().contains("a"));
assertTrue(nlinfo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));

// Remove all
params = new MultivaluedMapImpl();
Expand All @@ -303,7 +314,7 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains(""));
assertTrue(nlsifo.getNodeLabelsInfo().isEmpty());

// Add a label back for auth tests
params = new MultivaluedMapImpl();
Expand All @@ -326,7 +337,7 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("a"));
assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));

// Auth fail replace labels on node
params = new MultivaluedMapImpl();
Expand All @@ -347,7 +358,7 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("a"));
assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));

// Fail to add a label with post
response =
Expand Down Expand Up @@ -383,8 +394,11 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName());
assertEquals(1, nlsifo.getNodeLabels().size());
for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
assertEquals("a", nl.getName());
assertTrue(nl.getExclusivity());
}

// Remove cluster label with post
params = new MultivaluedMapImpl();
Expand All @@ -409,8 +423,8 @@ public void testNodeLabels() throws JSONException, Exception {
// configuration is on
// Reset for testing : add cluster labels
nlsifo = new NodeLabelsInfo();
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x"));
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y"));
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x", false));
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y", false));
response =
r.path("ws")
.path("v1")
Expand All @@ -421,8 +435,6 @@ public void testNodeLabels() throws JSONException, Exception {
.entity(toJson(nlsifo, NodeLabelsInfo.class),
MediaType.APPLICATION_JSON).post(ClientResponse.class);
// Reset for testing : Add labels to a node
nlsifo = new NodeLabelsInfo();
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y"));
params = new MultivaluedMapImpl();
params.add("labels", "y");
response =
Expand Down Expand Up @@ -461,7 +473,8 @@ public void testNodeLabels() throws JSONException, Exception {
ntlinfo = response.getEntity(NodeToLabelsInfo.class);
nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size());
assertFalse(nlinfo.getNodeLabels().contains("x"));
assertFalse(nlinfo.getNodeLabelsInfo().contains(
new NodeLabelInfo("x", false)));

// Case2 : failure to Replace labels using replace-labels
response =
Expand All @@ -481,7 +494,8 @@ public void testNodeLabels() throws JSONException, Exception {
ntlinfo = response.getEntity(NodeToLabelsInfo.class);
nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size());
assertFalse(nlinfo.getNodeLabels().contains("x"));
assertFalse(nlinfo.getNodeLabelsInfo().contains(
new NodeLabelInfo("x", false)));

// Case3 : Remove cluster label should be successful
params = new MultivaluedMapImpl();
Expand All @@ -500,7 +514,10 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals(new NodeLabelInfo("y", false),
nlsifo.getNodeLabelsInfo().get(0));
assertEquals("y", nlsifo.getNodeLabelsInfo().get(0).getName());
assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());

// Remove y
params = new MultivaluedMapImpl();
Expand Down Expand Up @@ -543,7 +560,7 @@ public void testNodeLabels() throws JSONException, Exception {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals("z", nlsifo.getNodeLabelsInfo().get(0).getName());
assertEquals(false, nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
assertEquals(1, nlsifo.getNodeLabels().size());
}

Expand Down

0 comments on commit ec0a852

Please sign in to comment.