Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Closes OOZIE-130 build workflow progress information in Oozie
  • Loading branch information
Chao Wang committed Aug 1, 2011
1 parent 31586bb commit f7cab5e
Show file tree
Hide file tree
Showing 20 changed files with 1,421 additions and 3 deletions.
4 changes: 4 additions & 0 deletions client/src/main/java/org/apache/oozie/cli/OozieCLI.java
Expand Up @@ -878,6 +878,10 @@ private void printJob(WorkflowJob job, boolean localtime, boolean verbose) throw
System.out.println("Last Modified : " + maskDate(job.getLastModifiedTime(), localtime));
System.out.println("Ended : " + maskDate(job.getEndTime(), localtime));
System.out.println("CoordAction ID: " + maskIfNull(job.getParentId()));
if (job.getProgress() >= 0) { // -1.0f means no progress is available, therefore do not display.
System.out.printf("%% Complete : %.2f%%\n", job.getProgress() * 100f);

This comment has been minimized.

Copy link
@mislam77-zz

mislam77-zz Aug 4, 2011

Collaborator

Why it is displaying conditionally? It will make the output inconsistent. For example, if someone is parsing this output, he will need to consider the presence and non-presence of this field. Moreover, when we will add more columns, it will be hard to isolate this column. If there is no work done, could not we show 0%?

This comment has been minimized.

Copy link
@brookwc

brookwc Aug 5, 2011

Owner

Will remote this. Originally it's for the cases where no progress info is available from oozie server.
But for now, all wfs will have some progress score - so can be simply removed.

If there is no work done, should show 100%, rather 0%.

}


List<WorkflowAction> actions = job.getActions();

Expand Down
Expand Up @@ -135,5 +135,10 @@ public static enum Status {
* @return the workflow nodes that already executed and are executing.
*/
List<WorkflowAction> getActions();


/**
* Return the workflow job progress in %.
*/
float getProgress();

}
Expand Up @@ -45,6 +45,7 @@ public interface JsonTags {
public static final String WORKFLOW_RUN = "run";
public static final String WORKFLOW_CONSOLE_URL = "consoleUrl";
public static final String WORKFLOW_ACTIONS = "actions";
public static final String WORKFLOW_PROGRESS = "progress";

public static final String WORKFLOWS_JOBS = "workflows";
public static final String WORKFLOWS_TOTAL = "total";
Expand Down
Expand Up @@ -94,6 +94,7 @@ public Property(String label, Class type, boolean isList) {
WF_JOB.put("getRun", new Property(JsonTags.WORKFLOW_RUN, Integer.TYPE));
WF_JOB.put("getConsoleUrl", new Property(JsonTags.WORKFLOW_CONSOLE_URL, String.class));
WF_JOB.put("getActions", new Property(JsonTags.WORKFLOW_ACTIONS, WorkflowAction.class, true));
WF_JOB.put("getProgress", new Property(JsonTags.WORKFLOW_PROGRESS, Float.TYPE));
WF_JOB.put("getParentId", new Property(JsonTags.WORKFLOW_PARENT_ID, String.class));
WF_JOB.put("toString", new Property(JsonTags.TO_STRING, String.class));

Expand Down Expand Up @@ -216,6 +217,9 @@ else if (type == Integer.TYPE) {
else if (type == Long.TYPE) {
return (obj != null) ? obj : new Long(0);
}
else if (type == Float.TYPE) {
return (obj != null) ? new Float(((Double) obj).floatValue()) : new Float(-1);

This comment has been minimized.

Copy link
@tucu00

tucu00 Aug 23, 2011

Collaborator

The value here, to be consistent with other primitive types must be ZERO.

This comment has been minimized.

Copy link
@brookwc

brookwc Sep 6, 2011

Owner

later review feedback commit already removed -1 piece.

}
else if (type == Date.class) {
return JsonUtils.parseDateRfc822((String) obj);
}
Expand Down
14 changes: 14 additions & 0 deletions client/src/main/java/org/apache/oozie/client/rest/JsonUtils.java
Expand Up @@ -65,5 +65,19 @@ static Date parseDateRfc822(String str) {
}
return null;
}

/**
* Return a float value from a JSONObject.
* This is used only for getting job's progress value.
*
* @param map JSON object.
* @param name name of the property.
* @return the float value associated with it, or -1.0 if not defined.
*/
public static float getFloatValue(JSONObject map, String name) {
Double d = (Double) map.get(name);

return (d != null) ? d.floatValue() : -1.0f;

This comment has been minimized.

Copy link
@tucu00

tucu00 Aug 23, 2011

Collaborator

The value here, to be consistent with primitive types must be ZERO.

This comment has been minimized.

Copy link
@brookwc

brookwc Sep 6, 2011

Owner

same here - this has become orphan code - already removed in later review commit.

}

}
2 changes: 2 additions & 0 deletions core/src/main/java/org/apache/oozie/WorkflowJobBean.java
Expand Up @@ -146,6 +146,7 @@ public void write(DataOutput dataOutput) throws IOException {
WritableUtils.writeStr(dataOutput, authToken);
WritableUtils.writeStr(dataOutput, logToken);
WritableUtils.writeStr(dataOutput, protoActionConf);
dataOutput.writeFloat(getProgress());
}

/**
Expand Down Expand Up @@ -186,6 +187,7 @@ public void readFields(DataInput dataInput) throws IOException {
protoActionConf = WritableUtils.readStr(dataInput);
setExternalId(getExternalId());
setProtoActionConf(protoActionConf);
setProgress(dataInput.readFloat());
}

public String getAuthToken() {
Expand Down
Expand Up @@ -91,6 +91,9 @@ public class JsonWorkflowJob implements WorkflowJob, JsonBean {
@Transient
private List<? extends JsonWorkflowAction> actions;

@Transient
private float progress;

public JsonWorkflowJob() {
actions = new ArrayList<JsonWorkflowAction>();
}
Expand All @@ -114,6 +117,7 @@ public JSONObject toJSONObject() {
json.put(JsonTags.WORKFLOW_RUN, (long) getRun());
json.put(JsonTags.WORKFLOW_CONSOLE_URL, getConsoleUrl());
json.put(JsonTags.WORKFLOW_ACTIONS, JsonWorkflowAction.toJSONArray(actions));
json.put(JsonTags.WORKFLOW_PROGRESS, progress);
json.put(JsonTags.TO_STRING, toString());
return json;
}
Expand Down Expand Up @@ -263,6 +267,14 @@ public List<WorkflowAction> getActions() {
return (List) actions;
}

public void setProgress(float p) {
progress = p;
}

public float getProgress() {
return progress;
}

public void setActions(List<? extends JsonWorkflowAction> nodes) {
this.actions = (nodes != null) ? nodes : new ArrayList<JsonWorkflowAction>();
}
Expand Down
49 changes: 48 additions & 1 deletion core/src/main/java/org/apache/oozie/command/wf/JobCommand.java
Expand Up @@ -14,12 +14,19 @@
*/
package org.apache.oozie.command.wf;

import java.util.List;

import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.action.decision.DecisionActionExecutor;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.WorkflowJob;
import org.apache.oozie.service.Services;
import org.apache.oozie.store.StoreException;
import org.apache.oozie.store.WorkflowStore;
import org.apache.oozie.util.ParamChecker;
import org.apache.oozie.util.XLog;
import org.apache.oozie.service.Services;
import org.apache.oozie.workflow.lite.LiteWorkflowApp;
import org.apache.oozie.workflow.lite.LiteWorkflowInstance;

/**
* Command for loading a job information
Expand Down Expand Up @@ -52,6 +59,18 @@ public JobCommand(String id, int start, int length) {
protected WorkflowJobBean call(WorkflowStore store) throws StoreException {
WorkflowJobBean workflow = store.getWorkflowInfoWithActionsSubset(id, start, len);
workflow.setConsoleUrl(getJobConsoleUrl(id));

// Estimate job progress
if (workflow.getStatus() == WorkflowJob.Status.PREP) {
workflow.setProgress(0.0f);
}
else if (workflow.getStatus() == WorkflowJob.Status.SUCCEEDED) {
workflow.setProgress(1.0f);
}
else {
workflow.setProgress(getJobProgress(workflow));
}

return workflow;
}

Expand All @@ -60,4 +79,32 @@ static String getJobConsoleUrl(String jobId) {
return (consoleUrl != null) ? consoleUrl + jobId : null;
}

/**
* Compute job progress that is defined as fraction of done actions.
*
* @param wf workflow job bean
* @return job progress
*/
static float getJobProgress(WorkflowJobBean wf) {
LiteWorkflowInstance wfInstance = (LiteWorkflowInstance) wf.getWorkflowInstance();
LiteWorkflowApp wfApp = (LiteWorkflowApp) wfInstance.getApp();

int executionPathLengthEstimate = wfApp.getExecutionPathLengthEstimate();
if (executionPathLengthEstimate == 0) { // noop wf
return 1.0f;
}
List<WorkflowAction> actions = wf.getActions();
int doneActions = 0;
for (WorkflowAction action : actions) {
// Skip decision nodes, note start, kill, end, fork/join will not have action entry.
if (action.getType().equals(DecisionActionExecutor.ACTION_TYPE)) {
continue;
}
if (action.getStatus() == WorkflowAction.Status.OK || action.getStatus() == WorkflowAction.Status.DONE) {
doneActions++;
}
}

return (doneActions * 1.0f) / executionPathLengthEstimate;
}
}
47 changes: 47 additions & 0 deletions core/src/main/java/org/apache/oozie/command/wf/JobXCommand.java
Expand Up @@ -14,15 +14,22 @@
*/
package org.apache.oozie.command.wf;

import java.util.List;

import org.apache.oozie.ErrorCode;
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.action.decision.DecisionActionExecutor;
import org.apache.oozie.command.CommandException;
import org.apache.oozie.command.PreconditionException;
import org.apache.oozie.executor.jpa.JPAExecutorException;
import org.apache.oozie.executor.jpa.WorkflowInfoWithActionsSubsetGetJPAExecutor;
import org.apache.oozie.service.JPAService;
import org.apache.oozie.service.Services;
import org.apache.oozie.util.ParamChecker;
import org.apache.oozie.workflow.lite.LiteWorkflowApp;
import org.apache.oozie.workflow.lite.LiteWorkflowInstance;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.WorkflowJob;

/**
* This Xcommand is returning the workflow with action within the range.
Expand Down Expand Up @@ -63,6 +70,17 @@ protected WorkflowJobBean execute() throws CommandException {
throw new CommandException(ErrorCode.E0610, this.id);
}
this.workflow.setConsoleUrl(getJobConsoleUrl(id));

// Estimate job progress
if (workflow.getStatus() == WorkflowJob.Status.PREP) {
workflow.setProgress(0.0f);
}
else if (workflow.getStatus() == WorkflowJob.Status.SUCCEEDED) {
workflow.setProgress(1.0f);
}
else {
workflow.setProgress(getJobProgress(workflow));
}
}
catch (JPAExecutorException ex) {
throw new CommandException(ex);
Expand Down Expand Up @@ -112,4 +130,33 @@ protected void loadState() throws CommandException {
@Override
protected void verifyPrecondition() throws CommandException, PreconditionException {
}

/**
* Compute job progress that is defined as fraction of done actions.
*
* @param wf workflow job bean
* @return job progress
*/
static float getJobProgress(WorkflowJobBean wf) {
LiteWorkflowInstance wfInstance = (LiteWorkflowInstance) wf.getWorkflowInstance();
LiteWorkflowApp wfApp = (LiteWorkflowApp) wfInstance.getApp();

int executionPathLengthEstimate = wfApp.getExecutionPathLengthEstimate();
if (executionPathLengthEstimate == 0) { // noop wf
return 1.0f;

This comment has been minimized.

Copy link
@tucu00

tucu00 Aug 23, 2011

Collaborator

don't do a return in the middle o the method, difficult to follow flow, return should be only one at the end of the method

This comment has been minimized.

Copy link
@brookwc

brookwc Sep 6, 2011

Owner

fixed.

}
List<WorkflowAction> actions = wf.getActions();
int doneActions = 0;
for (WorkflowAction action : actions) {
// Skip decision nodes, note start, kill, end, fork/join will not have action entry.
if (action.getType().equals(DecisionActionExecutor.ACTION_TYPE)) {
continue;

This comment has been minimized.

Copy link
@tucu00

tucu00 Aug 23, 2011

Collaborator

don't do continue, instead use an if block to skip

This comment has been minimized.

Copy link
@brookwc

brookwc Sep 6, 2011

Owner

fixed

}
if (action.getStatus() == WorkflowAction.Status.OK || action.getStatus() == WorkflowAction.Status.DONE) {

This comment has been minimized.

Copy link
@mislam77-zz

mislam77-zz Aug 4, 2011

Collaborator

DONE is bot a terminal state. So I believe it should not be counted as completed.

What happened to other states : KILLED, FAILED, ERROR? Does progress count only the SUCCESS case?

This comment has been minimized.

Copy link
@brookwc

brookwc Aug 5, 2011

Owner

you're right. I only considered success state here - consider all terminal states make more sense.

Done looks to me a terminal state, no?

doneActions++;
}
}

return (doneActions * 1.0f) / executionPathLengthEstimate;
}
}
Expand Up @@ -38,6 +38,7 @@ public class LiteWorkflowApp implements Writable, WorkflowApp {
private String definition;
private Map<String, NodeDef> nodesMap = new LinkedHashMap<String, NodeDef>();
private boolean complete = false;
private int executionPathLengthEstimate = -1;

LiteWorkflowApp() {
}
Expand Down Expand Up @@ -122,6 +123,7 @@ public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeUTF(n.getClass().getName());
n.write(dataOutput);
}
dataOutput.writeInt(executionPathLengthEstimate);
}

/**
Expand Down Expand Up @@ -176,6 +178,16 @@ public void readFields(DataInput dataInput) throws IOException {
throw new IOException(e);
}
}

executionPathLengthEstimate = dataInput.readInt();
}

public int getExecutionPathLengthEstimate() {
return executionPathLengthEstimate;
}

public void setExecutionPathLengthEstimate(int length) {
executionPathLengthEstimate = length;
}

}

1 comment on commit f7cab5e

@tucu00
Copy link
Collaborator

@tucu00 tucu00 commented on f7cab5e Aug 23, 2011

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn't be easier to for users if the progress is an integer representing the percentage, then is always 2 digits, easier to display

Please sign in to comment.