Permalink
Browse files

Merge branch 'model_refactor' of github.com:twitter/ambrose into hack…

…week
  • Loading branch information...
2 parents 74d23a0 + 5374520 commit 234c97d95279a78b3e105ee844c4540459adb507 @sagemintblue sagemintblue committed Apr 5, 2013
@@ -23,6 +23,7 @@
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@@ -42,29 +43,21 @@
public class DAGNode<T extends Job> {
private String name;
private T job;
-
- private Collection<DAGNode> successors;
+ private Collection<DAGNode<? extends Job>> successors;
private Collection<String> successorNames;
- public DAGNode(String name, T job) {
- this.name = name;
- this.job = job;
- }
-
@JsonCreator
public DAGNode(@JsonProperty("name") String name,
- @JsonProperty("job") T job,
- @JsonProperty("successorNames") Collection<String> successorNames) {
+ @JsonProperty("job") T job) {
this.name = name;
- this.successorNames = successorNames;
+ this.job = job;
}
public String getName() { return name; }
public T getJob() { return job; }
- @JsonIgnore
- public synchronized Collection<DAGNode> getSuccessors() { return successors;}
- public synchronized void setSuccessors(Collection<DAGNode> successors) {
+ public synchronized Collection<DAGNode<? extends Job>> getSuccessors() { return successors; }
+ public synchronized void setSuccessors(Collection<DAGNode<? extends Job>> successors) {
Collection<String> successorNames = new HashSet<String>();
if (successors != null) {
for(DAGNode node : successors) {
@@ -77,6 +70,14 @@ public synchronized void setSuccessors(Collection<DAGNode> successors) {
public synchronized Collection<String> getSuccessorNames() { return successorNames; }
+ public String toJson() throws IOException {
+ return JSONUtil.toJson(this);
+ }
+
+ public static DAGNode<? extends Job> fromJson(String json) throws IOException {
+ return JSONUtil.toObject(json, new TypeReference<DAGNode<? extends Job>>() { });
+ }
+
@SuppressWarnings("unchecked")
public static void main(String[] args) throws IOException {
String sourceFile = "pig/src/main/resources/web/data/large-dag.json";
@@ -15,6 +15,11 @@
*/
package com.twitter.ambrose.model;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.JsonTypeName;
import com.twitter.ambrose.util.JSONUtil;
import java.io.IOException;
@@ -31,6 +36,12 @@
*
* @author billg
*/
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
+@JsonTypeName("default")
+@JsonSubTypes({
+ @JsonSubTypes.Type(value=com.twitter.ambrose.model.Event.class, name="default"),
+ @JsonSubTypes.Type(value=com.twitter.ambrose.model.Event.JobStartedEvent.class, name="job_started"),
+})
public class Event<T> {
private static AtomicInteger NEXT_ID = new AtomicInteger();
@@ -45,6 +56,8 @@
private Type type;
private T payload;
+ protected Event(Type type) { this.type = type; }
+
private Event(Type type, T payload) {
this.id = NEXT_ID.incrementAndGet();
this.timestamp = System.currentTimeMillis();
@@ -62,16 +75,14 @@ public Event(int eventId, long timestamp, Type type, T payload) {
public long getTimestamp() { return timestamp; }
public int getId() { return id; }
public Type getType() { return type; }
- public Object getPayload() { return payload; }
+ public T getPayload() { return payload; }
- @SuppressWarnings("unchecked")
- public static void main(String[] args) throws IOException {
- String json = JSONUtil.readFile("pig/src/main/resources/web/data/small-events.json");
- List<Event> events = JSONUtil.toObject(json, new TypeReference<List<Event>>() { });
- for (Event event : events) {
- // useful if we need to read a file, add a field, output and re-generate
- }
- JSONUtil.writeJson("pig/src/main/resources/web/data/small-events.json2", events);
+ public String toJson() throws IOException {
+ return JSONUtil.toJson(this);
+ }
+
+ public static Event<?> fromJson(String json) throws IOException {
+ return JSONUtil.toObject(json, new TypeReference<Event<?>>() { });
}
/**
@@ -100,14 +111,20 @@ public static Event create(Type type, Object data) {
}
}
+ @JsonTypeName("job_started")
public static class JobStartedEvent extends Event<DAGNode<? extends Job>> {
+ @JsonCreator
+ public JobStartedEvent() { super(Type.JOB_STARTED); }
+
public JobStartedEvent(DAGNode<? extends Job> eventData) {
super(Type.JOB_STARTED, eventData);
}
}
public static class JobProgressEvent extends Event<DAGNode<? extends Job>> {
- public JobProgressEvent(DAGNode<? extends Job> eventData) {
+
+
+ public JobProgressEvent(@JsonProperty("payload") DAGNode<? extends Job> eventData) {
super(Type.JOB_PROGRESS, eventData);
}
}
@@ -129,4 +146,14 @@ public WorkflowProgressEvent(Map<WorkflowProgressField, String> eventData) {
super(Type.WORKFLOW_PROGRESS, eventData);
}
}
+
+ @SuppressWarnings("unchecked")
+ public static void main(String[] args) throws IOException {
+ String json = JSONUtil.readFile("pig/src/main/resources/web/data/small-events.json");
+ List<Event> events = JSONUtil.toObject(json, new TypeReference<List<Event>>() { });
+ for (Event event : events) {
+ // useful if we need to read a file, add a field, output and re-generate
+ }
+ JSONUtil.writeJson("pig/src/main/resources/web/data/small-events.json2", events);
+ }
}
@@ -15,10 +15,16 @@
*/
package com.twitter.ambrose.model;
-import org.codehaus.jackson.annotate.JsonCreator;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.twitter.ambrose.util.JSONUtil;
+
+import java.io.IOException;
import java.util.Map;
import java.util.Properties;
@@ -29,23 +35,22 @@
*
* @author billg
*/
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "runtime")
+@JsonTypeName("default")
+@JsonSubTypes({
+ @JsonSubTypes.Type(value=com.twitter.ambrose.model.Job.class, name="default"),
+})
public class Job {
-
private String id;
- private String runtime;
private Properties configuration;
private Map<String, Number> metrics;
- public Job(String runtime) {
- this(runtime, null, null, null);
- }
+ protected Job() { }
@JsonCreator
- public Job(@JsonProperty("runtime") String runtime,
- @JsonProperty("id") String id,
+ public Job(@JsonProperty("id") String id,
@JsonProperty("metrics") Map<String, Number> metrics,
@JsonProperty("configuration") Properties configuration) {
- this.runtime = runtime;
this.id = id;
this.metrics = metrics;
this.configuration = configuration;
@@ -54,11 +59,17 @@ public Job(@JsonProperty("runtime") String runtime,
public String getId() { return id; }
public void setId(String id) { this.id = id; }
- public String getRuntime() { return runtime; }
-
public Properties getConfiguration() { return configuration; }
public void setConfiguration(Properties configuration) { this.configuration = configuration; }
public Map<String, Number> getMetrics() { return metrics; }
protected void setMetrics(Map<String, Number> metrics) { this.metrics = metrics; }
+
+ public String toJson() throws IOException {
+ return JSONUtil.toJson(this);
+ }
+
+ public static Job fromJson(String json) throws IOException {
+ return JSONUtil.toObject(json, new TypeReference<Job>() { });
+ }
}
@@ -18,16 +18,15 @@
import java.io.IOException;
import java.util.List;
-import org.codehaus.jackson.annotate.JsonCreator;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.DeserializationConfig;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.SerializationConfig;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.twitter.ambrose.util.JSONUtil;
/**
* Class that represents the runtime stats for a given workflow. A workflow consists of 1 or more
- * jobs with dependancies that are run to produce a result. Basically a workflow is a collection of
- * jobs aranged to for a DAG. A Pig script or a Cascading flow are both examples of a workflow.
+ * jobs with dependencies that are run to produce a result. Basically a workflow is a collection of
+ * jobs arranged to for a DAG. A Pig script or a Cascading flow are both examples of a workflow.
*
* @author billg
*/
@@ -68,9 +67,7 @@ public Workflow(@JsonProperty("workflowId") String workflowId,
* @throws IOException
*/
public static String toJSON(Workflow workflow) throws IOException {
- ObjectMapper om = new ObjectMapper();
- om.getSerializationConfig().set(SerializationConfig.Feature.INDENT_OUTPUT, true);
- return om.writeValueAsString(workflow);
+ return JSONUtil.toJson(workflow);
}
/**
@@ -82,8 +79,6 @@ public static String toJSON(Workflow workflow) throws IOException {
* @throws IOException
*/
public static Workflow fromJSON(String workflowInfoJson) throws IOException {
- ObjectMapper om = new ObjectMapper();
- om.getDeserializationConfig().set(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- return om.readValue(workflowInfoJson, Workflow.class);
+ return JSONUtil.toObject(workflowInfoJson, new TypeReference<Workflow>() {});
}
}
@@ -18,10 +18,10 @@
import java.util.HashMap;
import java.util.Map;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.Counters.Counter;
-import org.codehaus.jackson.annotate.JsonCreator;
-import org.codehaus.jackson.annotate.JsonProperty;
/**
* Immutable class that represents a group of Hadoop counters along with the individual counter
@@ -1,5 +1,6 @@
package com.twitter.ambrose.model.hadoop;
+import com.fasterxml.jackson.annotation.JsonCreator;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TaskReport;
@@ -19,6 +20,9 @@
private int totalMappers;
private int totalReducers;
+ @JsonCreator
+ public MapReduceJobState() { }
+
@SuppressWarnings("deprecation")
public MapReduceJobState(RunningJob runningJob,
TaskReport[] mapTaskReport,
@@ -38,35 +42,71 @@ public String getJobId() {
return jobId;
}
+ public void setJobId(String jobId) {
+ this.jobId = jobId;
+ }
+
public String getJobName() {
return jobName;
}
+ public void setJobName(String jobName) {
+ this.jobName = jobName;
+ }
+
public String getTrackingURL() {
return trackingURL;
}
+ public void setTrackingURL(String trackingURL) {
+ this.trackingURL = trackingURL;
+ }
+
public boolean isComplete() {
return isComplete;
}
+ public void setComplete(boolean complete) {
+ isComplete = complete;
+ }
+
public boolean isSuccessful() {
return isSuccessful;
}
+ public void setSuccessful(boolean successful) {
+ isSuccessful = successful;
+ }
+
public float getMapProgress() {
return mapProgress;
}
+ public void setMapProgress(float mapProgress) {
+ this.mapProgress = mapProgress;
+ }
+
public float getReduceProgress() {
return reduceProgress;
}
+ public void setReduceProgress(float reduceProgress) {
+ this.reduceProgress = reduceProgress;
+ }
+
public int getTotalMappers() {
return totalMappers;
}
+ public void setTotalMappers(int totalMappers) {
+ this.totalMappers = totalMappers;
+ }
+
public int getTotalReducers() {
return totalReducers;
}
+
+ public void setTotalReducers(int totalReducers) {
+ this.totalReducers = totalReducers;
+ }
}
@@ -108,4 +108,8 @@ public static String readFile(String path) throws IOException {
mapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS);
mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
}
+
+ public static void mixinAnnotatons(Class<?> target, Class<?> mixinSource) {
+ mapper.addMixInAnnotations(target, mixinSource);
+ }
}
Oops, something went wrong.

0 comments on commit 234c97d

Please sign in to comment.