Skip to content

Commit

Permalink
YARN-3379. Fixed missing data in localityTable and ResourceRequests t…
Browse files Browse the repository at this point in the history 
…able in RM WebUI. Contributed by Xuan Gong
  • Loading branch information
@jian-he
jian-he committed Mar 20, 2015
1 parent e37ca22 commit 4e886eb
Show file tree
Hide file tree
Showing 7 changed files with 290 additions and 148 deletions.
3 changes: 3 additions & 0 deletions hadoop-yarn-project/CHANGES.txt
View file
Expand Up @@ -796,6 +796,9 @@ Release 2.7.0 - UNRELEASED
YARN-3349. Treat all exceptions as failure in
TestFSRMStateStore#testFSRMStateStoreClientRetry. (Zhihai Xu via ozawa)

YARN-3379. Fixed missing data in localityTable and ResourceRequests table
in RM WebUI. (Xuan Gong via jianhe)

Release 2.6.0 - 2014-11-18

INCOMPATIBLE CHANGES
Expand Down
71 changes: 5 additions & 66 deletions ...arn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
View file
Expand Up @@ -19,12 +19,6 @@

import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ATTEMPT_ID;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._EVEN;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._ODD;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;

import java.security.PrivilegedExceptionAction;
import java.util.Collection;

Expand All @@ -43,20 +37,18 @@
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;

import com.google.inject.Inject;

public class AppAttemptBlock extends HtmlBlock {

private static final Log LOG = LogFactory.getLog(AppAttemptBlock.class);
protected ApplicationBaseProtocol appBaseProt;
protected ApplicationAttemptId appAttemptId = null;

@Inject
public AppAttemptBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
Expand All @@ -66,14 +58,12 @@ public AppAttemptBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {

@Override
protected void render(Block html) {
String webUiType = $(WEB_UI_TYPE);
String attemptid = $(APPLICATION_ATTEMPT_ID);
if (attemptid.isEmpty()) {
puts("Bad request: requires application attempt ID");
return;
}

ApplicationAttemptId appAttemptId = null;
try {
appAttemptId = ConverterUtils.toApplicationAttemptId(attemptid);
} catch (IllegalArgumentException e) {
Expand Down Expand Up @@ -183,17 +173,7 @@ public Collection<ContainerReport> run() throws Exception {
return;
}

// TODO need to render applicationHeadRoom value from
// ApplicationAttemptMetrics after YARN-3284
if (webUiType.equals(YarnWebParams.RM_WEB_UI)) {
if (!isApplicationInFinalState(appAttempt.getAppAttemptState())) {
DIV<Hamlet> pdiv = html._(InfoBlock.class).div(_INFO_WRAP);
info("Application Attempt Overview").clear();
info("Application Attempt Metrics")._(
"Application Attempt Headroom : ", 0);
pdiv._();
}
}
createAttemptHeadRoomTable(html);
html._(InfoBlock.class);

// Container Table
Expand Down Expand Up @@ -236,45 +216,6 @@ public Collection<ContainerReport> run() throws Exception {
._("var containersTableData=" + containersTableData)._();

tbody._()._();

if (webUiType.equals(YarnWebParams.RM_WEB_UI)) {
createContainerLocalityTable(html); // TODO:YARN-3284
}
}

//TODO: YARN-3284
//The containerLocality metrics will be exposed from AttemptReport
private void createContainerLocalityTable(Block html) {
int totalAllocatedContainers = 0; //TODO: YARN-3284
int[][] localityStatistics = new int[0][0];//TODO:YARN-3284
DIV<Hamlet> div = html.div(_INFO_WRAP);
TABLE<DIV<Hamlet>> table =
div.h3(
"Total Allocated Containers: "
+ totalAllocatedContainers).h3("Each table cell"
+ " represents the number of NodeLocal/RackLocal/OffSwitch containers"
+ " satisfied by NodeLocal/RackLocal/OffSwitch resource requests.").table(
"#containerLocality");
table.
tr().
th(_TH, "").
th(_TH, "Node Local Request").
th(_TH, "Rack Local Request").
th(_TH, "Off Switch Request").
_();

String[] containersType =
{ "Num Node Local Containers (satisfied by)", "Num Rack Local Containers (satisfied by)",
"Num Off Switch Containers (satisfied by)" };
boolean odd = false;
for (int i = 0; i < localityStatistics.length; i++) {
table.tr((odd = !odd) ? _ODD : _EVEN).td(containersType[i])
.td(String.valueOf(localityStatistics[i][0]))
.td(i == 0 ? "" : String.valueOf(localityStatistics[i][1]))
.td(i <= 1 ? "" : String.valueOf(localityStatistics[i][2]))._();
}
table._();
div._();
}

private boolean hasAMContainer(ContainerId containerId,
Expand All @@ -286,10 +227,8 @@ private boolean hasAMContainer(ContainerId containerId,
}
return false;
}

private boolean isApplicationInFinalState(YarnApplicationAttemptState state) {
return state == YarnApplicationAttemptState.FINISHED
|| state == YarnApplicationAttemptState.FAILED
|| state == YarnApplicationAttemptState.KILLED;

protected void createAttemptHeadRoomTable(Block html) {

}
}
87 changes: 9 additions & 78 deletions ...adoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
View file
Expand Up @@ -21,11 +21,8 @@
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import java.util.List;

import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
Expand All @@ -39,12 +36,9 @@
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
Expand All @@ -53,10 +47,8 @@
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
Expand All @@ -69,9 +61,11 @@ public class AppBlock extends HtmlBlock {
private static final Log LOG = LogFactory.getLog(AppBlock.class);
protected ApplicationBaseProtocol appBaseProt;
protected Configuration conf;
protected ApplicationId appID = null;

@Inject
AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx, Configuration conf) {
protected AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx,
Configuration conf) {
super(ctx);
this.appBaseProt = appBaseProt;
this.conf = conf;
Expand All @@ -86,7 +80,6 @@ protected void render(Block html) {
return;
}

ApplicationId appID = null;
try {
appID = Apps.toAppID(aid);
} catch (Exception e) {
Expand Down Expand Up @@ -213,31 +206,7 @@ public Collection<ApplicationAttemptReport> run() throws Exception {
return;
}

//TODO:YARN-3284
//The preemption metrics will be exposed from ApplicationReport
// and ApplicationAttemptReport
ApplicationResourceUsageReport usageReport =
appReport.getApplicationResourceUsageReport();
DIV<Hamlet> pdiv = html.
_(InfoBlock.class).
div(_INFO_WRAP);
info("Application Overview").clear();
info("Application Metrics")
._("Total Resource Preempted:",
Resources.none()) // TODO: YARN-3284
._("Total Number of Non-AM Containers Preempted:",
String.valueOf(0)) // TODO: YARN-3284
._("Total Number of AM Containers Preempted:",
String.valueOf(0)) // TODO: YARN-3284
._("Resource Preempted from Current Attempt:",
Resources.none()) // TODO: YARN-3284
._("Number of Non-AM Containers Preempted from Current Attempt:",
0) // TODO: YARN-3284
._("Aggregate Resource Allocation:",
String.format("%d MB-seconds, %d vcore-seconds", usageReport == null
? 0 : usageReport.getMemorySeconds(), usageReport == null ? 0
: usageReport.getVcoreSeconds()));
pdiv._();
createApplicationMetricsTable(html);

html._(InfoBlock.class);

Expand Down Expand Up @@ -319,49 +288,6 @@ public ContainerReport run() throws Exception {
._("var attemptsTableData=" + attemptsTableData)._();

tbody._()._();

if (webUiType != null && webUiType.equals(YarnWebParams.RM_WEB_UI)) {
createResourceRequestsTable(html, null); // TODO:YARN-3284
}
}

//TODO:YARN-3284
//The resource requests metrics will be exposed from attemptReport
private void createResourceRequestsTable(Block html, List<ResourceRequest> resouceRequests) {
TBODY<TABLE<Hamlet>> tbody =
html.table("#ResourceRequests").thead().tr()
.th(".priority", "Priority")
.th(".resourceName", "ResourceName")
.th(".totalResource", "Capability")
.th(".numContainers", "NumContainers")
.th(".relaxLocality", "RelaxLocality")
.th(".nodeLabelExpression", "NodeLabelExpression")._()._().tbody();

Resource totalResource = Resource.newInstance(0, 0);
if (resouceRequests != null) {
for (ResourceRequest request : resouceRequests) {
if (request.getNumContainers() == 0) {
continue;
}

tbody.tr()
.td(String.valueOf(request.getPriority()))
.td(request.getResourceName())
.td(String.valueOf(request.getCapability()))
.td(String.valueOf(request.getNumContainers()))
.td(String.valueOf(request.getRelaxLocality()))
.td(request.getNodeLabelExpression() == null ? "N/A" : request
.getNodeLabelExpression())._();
if (request.getResourceName().equals(ResourceRequest.ANY)) {
Resources.addTo(totalResource,
Resources.multiply(request.getCapability(),
request.getNumContainers()));
}
}
}
html.div().$class("totalResourceRequests")
.h3("Total Outstanding Resource Requests: " + totalResource)._();
tbody._()._();
}

private String clarifyAppState(YarnApplicationState state) {
Expand Down Expand Up @@ -389,4 +315,9 @@ private String clairfyAppFinalStatus(FinalApplicationStatus status) {
}
return status.toString();
}

// The preemption metrics only need to be shown in RM WebUI
protected void createApplicationMetricsTable(Block html) {

}
}
3 changes: 1 addition & 2 deletions ...er/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
View file
Expand Up @@ -23,7 +23,6 @@
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;

import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
Expand Down Expand Up @@ -51,7 +50,7 @@ protected void preHead(Page.HTML<_> html) {

@Override
protected Class<? extends SubView> content() {
return AppAttemptBlock.class;
return RMAppAttemptBlock.class;
}

}
3 changes: 1 addition & 2 deletions ...cemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java
View file
Expand Up @@ -23,7 +23,6 @@
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;

import org.apache.hadoop.yarn.server.webapp.AppBlock;
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
Expand All @@ -50,6 +49,6 @@ protected void preHead(Page.HTML<_> html) {

@Override
protected Class<? extends SubView> content() {
return AppBlock.class;
return RMAppBlock.class;
}
}

0 comments on commit 4e886eb

Please sign in to comment.