Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

We’re showing branches in this repository, but you can also compare across forks.

base fork: twitter/ambrose
base: e7ff61bdf9
...
head fork: twitter/ambrose
compare: 19265b200b
  • 6 commits
  • 13 files changed
  • 0 commit comments
  • 2 contributors
6 common/src/main/java/com/twitter/ambrose/model/Job.java
View
@@ -29,7 +29,6 @@
*
* @author billg
*/
-@SuppressWarnings("deprecation")
@JsonSerialize(
include=JsonSerialize.Inclusion.NON_NULL
)
@@ -44,10 +43,6 @@ public Job(String runtime) {
this(runtime, null, null, null);
}
- public Job(String runtime, Properties configuration) {
- this(runtime, null, null, configuration);
- }
-
@JsonCreator
public Job(@JsonProperty("runtime") String runtime,
@JsonProperty("id") String id,
@@ -65,6 +60,7 @@ public Job(@JsonProperty("runtime") String runtime,
public String getRuntime() { return runtime; }
public Properties getConfiguration() { return configuration; }
+ public void setConfiguration(Properties configuration) { this.configuration = configuration; }
public Map<String, Number> getMetrics() { return metrics; }
protected void setMetrics(Map<String, Number> metrics) { this.metrics = metrics; }
72 common/src/main/java/com/twitter/ambrose/model/hadoop/MapReduceJobState.java
View
@@ -0,0 +1,72 @@
+package com.twitter.ambrose.model.hadoop;
+
+import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapred.TaskReport;
+
+import java.io.IOException;
+
+/**
+ * Container that holds state of a MapReduce job
+ */
+public class MapReduceJobState {
+ private String jobId;
+ private String jobName;
+ private String trackingURL;
+ private boolean isComplete;
+ private boolean isSuccessful;
+ private float mapProgress;
+ private float reduceProgress;
+ private int totalMappers;
+ private int totalReducers;
+
+ @SuppressWarnings("deprecation")
+ public MapReduceJobState(RunningJob runningJob,
+ TaskReport[] mapTaskReport,
+ TaskReport[] reduceTaskReport) throws IOException {
+ jobId = runningJob.getID().toString();
+ jobName = runningJob.getJobName();
+ trackingURL = runningJob.getTrackingURL();
+ isComplete = runningJob.isComplete();
+ isSuccessful = runningJob.isSuccessful();
+ mapProgress = runningJob.mapProgress();
+ reduceProgress = runningJob.reduceProgress();
+ totalMappers = mapTaskReport.length;
+ totalReducers = reduceTaskReport.length;
+ }
+
+ public String getJobId() {
+ return jobId;
+ }
+
+ public String getJobName() {
+ return jobName;
+ }
+
+ public String getTrackingURL() {
+ return trackingURL;
+ }
+
+ public boolean isComplete() {
+ return isComplete;
+ }
+
+ public boolean isSuccessful() {
+ return isSuccessful;
+ }
+
+ public float getMapProgress() {
+ return mapProgress;
+ }
+
+ public float getReduceProgress() {
+ return reduceProgress;
+ }
+
+ public int getTotalMappers() {
+ return totalMappers;
+ }
+
+ public int getTotalReducers() {
+ return totalReducers;
+ }
+}
6 common/src/main/java/com/twitter/ambrose/service/StatsReadService.java
View
@@ -15,6 +15,8 @@
*/
package com.twitter.ambrose.service;
+import com.twitter.ambrose.model.Job;
+
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
@@ -26,14 +28,14 @@
*
* @author billg
*/
-public interface StatsReadService {
+public interface StatsReadService<T extends Job> {
/**
* Get a map of all DAGNodes in the workflow.
* @param workflowId the id of the workflow being fetched
* @return a Map of DAGNodes where the key is the DAGNode name
*/
- public Map<String, DAGNode> getDagNodeNameMap(String workflowId) throws IOException;
+ public Map<String, DAGNode<T>> getDagNodeNameMap(String workflowId) throws IOException;
/**
* Get all events for a given workflow since eventId. To get the entire list of events, pass a
6 common/src/main/java/com/twitter/ambrose/service/StatsWriteService.java
View
@@ -15,6 +15,8 @@
*/
package com.twitter.ambrose.service;
+import com.twitter.ambrose.model.Job;
+
import java.io.IOException;
import java.util.Map;
@@ -25,7 +27,7 @@
*
* @author billg
*/
-public interface StatsWriteService {
+public interface StatsWriteService<T extends Job> {
/**
* Send a map of all DAGNodes in the workflow. The structure of the DAG is assumed to be immutable.
@@ -35,7 +37,7 @@
* @param workflowId the id of the workflow being updated
* @param dagNodeNameMap a Map of DAGNodes where the key is the DAGNode name
*/
- public void sendDagNodeNameMap(String workflowId, Map<String, DAGNode> dagNodeNameMap) throws IOException;
+ public void sendDagNodeNameMap(String workflowId, Map<String, DAGNode<T>> dagNodeNameMap) throws IOException;
/**
* Push an events for a given workflow.
11 common/src/main/java/com/twitter/ambrose/service/impl/InMemoryStatsService.java
View
@@ -15,6 +15,7 @@
*/
package com.twitter.ambrose.service.impl;
+import com.twitter.ambrose.model.Job;
import com.twitter.ambrose.service.DAGNode;
import com.twitter.ambrose.service.StatsReadService;
import com.twitter.ambrose.service.StatsWriteService;
@@ -51,13 +52,13 @@
*
* @author billg
*/
-public class InMemoryStatsService implements StatsReadService, StatsWriteService {
+public class InMemoryStatsService implements StatsReadService, StatsWriteService<Job> {
private static final Logger LOG = LoggerFactory.getLogger(InMemoryStatsService.class);
private static final String DUMP_DAG_FILE_PARAM = "ambrose.write.dag.file";
private static final String DUMP_EVENTS_FILE_PARAM = "ambrose.write.events.file";
- private Map<String, DAGNode> dagNodeNameMap = new HashMap<String, DAGNode>();
+ private Map<String, DAGNode<Job>> dagNodeNameMap = new HashMap<String, DAGNode<Job>>();
private SortedMap<Integer, WorkflowEvent> eventMap =
new ConcurrentSkipListMap<Integer, WorkflowEvent>();
@@ -86,12 +87,12 @@ public InMemoryStatsService() {
}
@Override
- public synchronized void sendDagNodeNameMap(String workflowId, Map<String, DAGNode> dagNodeNameMap) {
+ public synchronized void sendDagNodeNameMap(String workflowId, Map<String, DAGNode<Job>> dagNodeNameMap) {
this.dagNodeNameMap = dagNodeNameMap;
}
@Override
- public synchronized Map<String, DAGNode> getDagNodeNameMap(String workflowId) {
+ public synchronized Map<String, DAGNode<Job>> getDagNodeNameMap(String workflowId) {
return dagNodeNameMap;
}
@@ -111,7 +112,7 @@ public synchronized void pushEvent(String workflowId, WorkflowEvent event) {
public void writeJsonToDisk() throws IOException {
if (dagWriter != null && dagNodeNameMap != null) {
- Collection<DAGNode> nodes = getDagNodeNameMap(null).values();
+ Collection<DAGNode<Job>> nodes = getDagNodeNameMap(null).values();
JSONUtil.writeJson(dagWriter, nodes.toArray(new DAGNode[dagNodeNameMap.size()]));
dagWriter.close();
}
1,050 common/src/main/resources/web/data/small-dag.json
View
@@ -1,31 +1,1033 @@
[ {
"name" : "scope-44",
- "aliases" : [ "JOINED", "L", "R" ],
- "features" : [ "HASH_JOIN" ],
- "jobId" : "job_201204041958_199287",
- "successorNames" : [ "scope-48" ],
- "runtime" : "pig",
- "y" : -60.0,
- "x" : 0.0,
- "dagLevel" : 0
+ "job" : {
+ "id" : "job_201301170859_1562015",
+ "aliases" : [ "JOINED", "L", "R" ],
+ "features" : [ "HASH_JOIN" ],
+ "mapReduceJobState" : {
+ "successful" : true,
+ "jobId" : "job_201301170859_1562015",
+ "complete" : true,
+ "jobName" : "PigLatin:exec.pig",
+ "trackingURL" : "http://hadoop-dw-jt.smf1.twitter.com:50030/jobdetails.jsp?jobid=job_201301170859_1562015",
+ "mapProgress" : 1.0,
+ "reduceProgress" : 1.0,
+ "totalMappers" : 2,
+ "totalReducers" : 1
+ },
+ "metrics" : {
+ "proactiveSpillCountObjects" : 0,
+ "avgMapTime" : 3272,
+ "hdfsBytesWritten" : 91400,
+ "maxReduceTime" : 16984,
+ "minReduceTime" : 16984,
+ "SMMSpillCount" : 0,
+ "proactiveSpillCountRecs" : 0,
+ "numberReduces" : 1,
+ "avgReduceTime" : 16984,
+ "mapInputRecords" : 140,
+ "numberMaps" : 2,
+ "bytesWritten" : 0,
+ "maxMapTime" : 3273,
+ "recordWritten" : 0,
+ "mapOutputRecords" : 140,
+ "reduceOutputRecords" : 1500,
+ "reduceInputRecords" : 140,
+ "minMapTime" : 3272
+ },
+ "configuration" : {
+ "hadoop.id.str" : "billg",
+ "job.end.retry.interval" : "30000",
+ "io.bytes.per.checksum" : "512",
+ "mapred.job.tracker.retiredjobs.cache.size" : "1000",
+ "pig.script.features" : "1056",
+ "udf.import.list" : "eNpb85aBtbiIQTgrsSxRr7QkM0fPsagosdIns7ikovGS7MzjiXOZGRg9GViKM6tSKwoYGBg4ylmAJFcJg0Ryfq5eSXlmSUlqEZBOTMnPL9AryEzXK2GQxSWVk5-YApSXwyVfXJJflApUoISsIDUntSAjMa8kKbMoBdkUZbyKYEYxlDCI5xel6yUWJCZnpIKlkkozc0oy84CSMmgymbkFOXDpCgApYmRM",
+ "mapred.task.profile.reduces" : "0-2",
+ "mapreduce.jobtracker.staging.root.dir" : "/user",
+ "mapreduce.job.cache.files.visibilities" : "true",
+ "mapred.job.reuse.jvm.num.tasks" : "-1",
+ "pig.exec.reducers.max" : "999",
+ "dfs.block.access.token.lifetime" : "600",
+ "mapred.reduce.tasks.speculative.execution" : "false",
+ "mapred.job.name" : "PigLatin:exec.pig",
+ "dfs.permissions.supergroup" : "supergroup",
+ "hadoop.http.authentication.kerberos.keytab" : "${user.home}/hadoop.keytab",
+ "io.seqfile.sorter.recordlimit" : "1000000",
+ "user.dir" : "/home/billg/ws/git/ambrose",
+ "java.vm.version" : "19.1-b02",
+ "mapred.task.tracker.http.address" : "0.0.0.0:50060",
+ "dfs.namenode.delegation.token.renew-interval" : "86400000",
+ "hadoop.workaround.non.threadsafe.getpwuid" : "false",
+ "mapred.system.dir" : "/hadoop/mapred/system",
+ "fs.ramfs.impl" : "org.apache.hadoop.fs.InMemoryFileSystem",
+ "mapred.task.tracker.report.address" : "127.0.0.1:0",
+ "mapreduce.reduce.shuffle.connect.timeout" : "180000",
+ "sun.os.patch.level" : "unknown",
+ "pig.version" : "0.11.0",
+ "mapred.job.tracker.history.completed.location" : "hdfs://hadoop-dw-nn.smf1.twitter.com:8020/hadoop/mapred/history/done",
+ "mapred.healthChecker.interval" : "180000",
+ "java.vm.specification.name" : "Java Virtual Machine Specification",
+ "mapreduce.job.complete.cancel.delegation.tokens" : "true",
+ "fs.trash.interval" : "10080",
+ "mapred.skip.map.auto.incr.proc.count" : "true",
+ "hadoop.http.authentication.kerberos.principal" : "HTTP/localhost@LOCALHOST",
+ "mapred.fairscheduler.update.interval" : "5000",
+ "mapred.child.tmp" : "./tmp",
+ "os.name" : "Linux",
+ "mapred.tasktracker.taskmemorymanager.monitoring-interval" : "5000",
+ "dfs.datanode.http.address" : "0.0.0.0:50075",
+ "mapred.output.key.comparator.class" : "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler$PigCharArrayWritableComparator",
+ "io.sort.spill.percent" : "0.80",
+ "mapred.job.shuffle.input.buffer.percent" : "0.70",
+ "dfs.max.objects" : "0",
+ "java.vendor.url.bug" : "http://java.sun.com/cgi-bin/bugreport.cgi",
+ "mapred.skip.map.max.skip.records" : "0",
+ "mapreduce.reduce.shuffle.maxfetchfailures" : "10",
+ "hadoop.security.authorization" : "false",
+ "os.arch" : "amd64",
+ "proc_jar" : "",
+ "pig.notification.listener" : "com.twitter.ambrose.pig.EmbeddedAmbrosePigProgressNotificationListener",
+ "pig.alias" : "JOINED,L,R",
+ "hadoop.job.history.user.location" : "none",
+ "mapred.task.profile.maps" : "0-2",
+ "dfs.https.server.keystore.resource" : "ssl-server.xml",
+ "dfs.replication.interval" : "3",
+ "mapred.local.dir" : "/data/disk1/mapred/local,/data/disk2/mapred/local,/data/disk3/mapred/local,/data/disk4/mapred/local,/data/disk5/mapred/local,/data/disk6/mapred/local,/data/disk7/mapred/local,/data/disk8/mapred/local,/data/disk9/mapred/local,/data/disk10/mapred/local,/data/disk11/mapred/local,/data/disk12/mapred/local",
+ "mapred.merge.recordsBeforeProgress" : "10000",
+ "mapred.job.tracker.http.address" : "0.0.0.0:50030",
+ "user.country" : "US",
+ "mapred.compress.map.output" : "true",
+ "mapred.userlog.retain.hours" : "24",
+ "mapred.used.genericoptionsparser" : "true",
+ "mapred.tasktracker.reduce.tasks.maximum" : "3",
+ "mapred.create.symlink" : "yes",
+ "hadoop.security.uid.cache.secs" : "14400",
+ "hadoop.log.file" : "hadoop.log",
+ "fs.har.impl.disable.cache" : "true",
+ "mapred.cluster.map.memory.mb" : "-1",
+ "java.endorsed.dirs" : "/usr/java/jdk1.6.0_24/jre/lib/endorsed",
+ "dfs.data.dir" : "/data/disk1/dfs,/data/disk2/dfs,/data/disk3/dfs,/data/disk4/dfs,/data/disk5/dfs,/data/disk6/dfs,/data/disk7/dfs,/data/disk8/dfs,/data/disk9/dfs,/data/disk10/dfs,/data/disk11/dfs,/data/disk12/dfs",
+ "dfs.access.time.precision" : "3600000",
+ "dfs.replication.min" : "1",
+ "pig.map.output.dirs" : "",
+ "mapreduce.job.submithost" : "smf1-aai-15-sr3.prod.twitter.com",
+ "java.vm.specification.version" : "1.0",
+ "fs.checkpoint.dir" : "/data/hadoop-dw-snn",
+ "hcat.pig.inner.tuple.name" : "t",
+ "fs.s3n.impl" : "org.apache.hadoop.fs.s3native.NativeS3FileSystem",
+ "mapreduce.tasktracker.outofband.heartbeat" : "false",
+ "mapred.jobtracker.restart.recover" : "false",
+ "pig.script" : "TCA9IExPQUQgJ3dzL3Rlc3RfZGF0YS9waWcvZGF0YTEudHh0JyBBUyAob3duZXI6Y2hhcmFycmF5LHBldDpjaGFyYXJyYXksYWdlOmludCxwaG9uZTpjaGFyYXJyYXkpOwpSID0gTE9BRCAnd3MvdGVzdF9kYXRhL3BpZy9kYXRhMS50eHQnIEFTIChvd25lcjpjaGFyYXJyYXkscGV0OmNoYXJhcnJheSxhZ2U6aW50LHBob25lOmNoYXJhcnJheSk7CkpPSU5FRCA9IEpPSU4gTCBCWSBvd25lciwgUiBCWSBvd25lcjsKT1JERVJFRCA9IE9SREVSIEpPSU5FRCBCWSBMOjpvd25lcjsKcm1mIHRtcC9vdXRwdXQKLS1leHBsYWluIEpPSU5FRDsKU1RPUkUgT1JERVJFRCBJTlRPICd0bXAvb3V0cHV0JzsKZXhlYzsKcm1mIHRtcC9vdXRwdXQyClNUT1JFIEwgSU5UTyAndG1wL291dHB1dDInOwo=",
+ "ambrose.write.dag.file" : "ambrose_dag.json",
+ "java.vendor" : "Sun Microsystems Inc.",
+ "ambrose.post.script.sleep.seconds" : "60",
+ "heartbeat.recheck.interval" : "450000",
+ "hadoop.logfile.size" : "10000000",
+ "pig.hadoop.version" : "0.20.2.cdh3u2-t1",
+ "file.separator" : "/",
+ "mapred.fairscheduler.eventlog.enabled" : "false",
+ "pig.streaming.log.dir" : "hdfs://hadoop-dw-nn.smf1.twitter.com/tmp/temp-977624468/tmp-1644509149/_logs",
+ "pig.input.dirs" : "hdfs://hadoop-dw-nn.smf1.twitter.com/user/billg/ws/test_data/pig/data1.txt,hdfs://hadoop-dw-nn.smf1.twitter.com/user/billg/ws/test_data/pig/data1.txt",
+ "pig.log.file" : "pig.log",
+ "dfs.support.append" : "true",
+ "pig.script.id" : "9bc04e10-2389-4b38-b9bc-8baa08cd55aa",
+ "sun.java.launcher" : "SUN_STANDARD",
+ "mapred.inmem.merge.threshold" : "1000",
+ "ipc.client.connection.maxidletime" : "20000",
+ "fs.checkpoint.size" : "536870912",
+ "dfs.blockreport.intervalMsec" : "3600000",
+ "fs.s3.sleepTimeSeconds" : "10",
+ "java.library.path" : ":/usr/lib64:/usr/lib/hadoop-0.20/lib/native/Linux-amd64-64:/usr/lib64/mecab/java:/usr/lib64:/usr/lib/hadoop-0.20/lib/native/Linux-amd64-64:/usr/lib64/mecab/java:/usr/lib/hadoop-client-cdh3u2-t1/lib/native/Linux-amd64-64",
+ "dfs.client.block.write.retries" : "3",
+ "mapred.jobtracker.retirejob.interval" : "14400000",
+ "mapred.output.value.groupfn.class" : "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler$PigGroupingCharArrayWritableComparator",
+ "mapred.reduce.tasks" : "1",
+ "pig.sql.type" : "hcat",
+ "mapred.queue.names" : "default",
+ "sun.arch.data.model" : "64",
+ "io.seqfile.lazydecompress" : "true",
+ "dfs.https.enable" : "false",
+ "mapred.fairscheduler.preemption" : "true",
+ "exectype" : "MAPREDUCE",
+ "mapred.hosts.exclude" : "/etc/hadoop/conf/mapred.hosts.exclude",
+ "path.separator" : ":",
+ "dfs.replication" : "3",
+ "ipc.client.tcpnodelay" : "false",
+ "sun.io.unicode.encoding" : "UnicodeLittle",
+ "pig.script.info.enabled" : "true",
+ "pig.logfile" : "/var/log/pig/pig_1364953681893.log",
+ "pig.script.submitted.timestamp" : "1364953684934",
+ "pig.pretty.print.schema" : "true",
+ "mapred.acls.enabled" : "false",
+ "mapred.tasktracker.dns.nameserver" : "default",
+ "user.language" : "en",
+ "mapred.submit.replication" : "10",
+ "pig.command.line" : "LWYgL2hvbWUvYmlsbGcvd3MvdGVzdF9waWcvZXhlYy5waWc=",
+ "pig.reduce.stores" : "eNqlVN1OE0EUPi0UgRT_IDHGGIMajRfsgKkgcgUakuIqjSUxKTFx2J1uB2Znh5lZ2Xph4p2v4IVv4Ht473v4DHpmd0uh1njB3mxm5jvnfOc7P99_Qc1oWDikH6iXWi48n8sjFvrc2Pqj9rud91_uTgBk6mQSACoIfZvoyKOKBj3mKR55BzQ4YjL0ejRMEuWxjAXoJ5FMRlwipNc3PKDCp32mPc0EdY9U7CqmqU208Vq7bfwzKL5KFWY6cDnkhh4I9jxJpWW6CTUuQ5Z1oM7Nq1RYnpt0YJabvVjlBx9qYpsLZmHRR4qkoEiQIuGxEoQnxD23FQs2EGvc4Rg-wYQPUwaRMbWwPNZSJNFpBqSAknb-Q0czhkeS2lRj4Ou-k5EIKiPStprLCAHTJtG2KbuJhZuj7tvl00amwb-Irq3yNFB1KGa9AzUaBGncgTkuVWo3rXUxwiYsaHacMmNZ2KKaCsEEN_EWzGpmUOK9vnKaUsGpcTrVfFcEdGDh1mgeIbWU7KVKMEx4KkcZC1cLOVxXEddP-DYnkDuN2J6mAdMWHo56UkwODM4A0fJaojkmjgknQd5COakZHy4lqXXxyuOsoppJ28IiWGiNui9FJYWoZERUck5UotCHIQNpnUfkMYHqWHhxIcdvcoFd0RdHiu76zXNxvb8rWfFhMn7J-hYejG1TZ0YGZojbyBSO6_3_RkDoMEh1B6o8dPMRJCqfj1qmymfPltcAv_GTCj-MMD_cHZta074rdfb55-2vP-i3Cag0YdLwj6zwkm-R2Sy327pIwxcSnuG9BXUcwVTLtsVpNNiGRRufn8rdg0MWoPKqWiZVcUzujNOIJ95gX5RhqhgG57mLt69pzMqhmO6mMnCoMfO9XT5hQAvbvbBrnpGyRZbCkyUpPRN3Vzx7wi1uOS9IYmJjRSyL1dL62trq40Zj9am7WlpZbTSeLK-vNNaR740RvoMwQ5641GYCQY05JboP0wEWe1NH2L7z--M21ZViPyCi2G7ObtIxv_cPfZpuN7vli4PqmkFlfwD-ghSK",
+ "io.compression.codecs" : "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec",
+ "io.file.buffer.size" : "65536",
+ "mapred.map.tasks.speculative.execution" : "true",
+ "pig.spill.size.threshold" : "5000000",
+ "java.class.version" : "50.0",
+ "mapred.map.max.attempts" : "4",
+ "mapred.job.shuffle.merge.percent" : "0.66",
+ "fs.har.impl" : "org.apache.hadoop.fs.HarFileSystem",
+ "hadoop.security.authentication" : "simple",
+ "fs.s3.buffer.dir" : "${hadoop.tmp.dir}/s3",
+ "io.compression.codec.lzo.class" : "com.hadoop.compression.lzo.LzoCodec",
+ "mapred.skip.reduce.auto.incr.proc.count" : "true",
+ "dfs.http.address" : "0.0.0.0:50070",
+ "mapred.job.tracker.jobhistory.lru.cache.size" : "5",
+ "pig.job.submitted.timestamp" : "1364953689767",
+ "mapred.job.classpath.files" : "/tmp/temp-977624468/tmp-278376779/guava-12.0.1.jar",
+ "pig.inputs" : "eNqtUD1OwzAUfqRFFBYkOiAGhDhAbVjp1KUSKEJIGTug19R1H3ISY7-QwIDEMRi4Afdg5x7cAbsFIVUMDHiwLL_v971-wKZ30L_BOxQ1kxEj5_A-Jc_t0_vh8xu-dGDjHLqeHlRrASBpuuHeCaSjymmBFvOFEpa0oMIaQZUYk1GZVTmsTpJAkkJvHn4vsVAMe2l0kwZLLTN2VOphnNdlHlkMB2kQlithGYTl-Gs0bC3DxWI292dSLnBWVXYwawZlKXwxPxXcELNyIq8KWXvl5JSM0bLxkpXn6xkyLuXiI6BbDh321zp8W_1k76SwnRv0Poa_hUfoTKCXc-VGTnuG_uS3NrtU2pojIgvKBTKcrJeK25Km0pSjSfE-xPVLqFwxll2P19JNazJMpbginYUEqJW1PmZK_ncvUXLrb_btJwVnwW0",
+ "mapreduce.outputformat.class" : "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat",
+ "file.encoding.pkg" : "sun.io",
+ "sun.cpu.endian" : "little",
+ "dfs.client.block.write.locateFollowingBlock.retries" : "8",
+ "dfs.replication.considerLoad" : "true",
+ "dfs.block.access.token.enable" : "false",
+ "mapreduce.job.acl-view-job" : " ",
+ "mapred.job.queue.name" : "default",
+ "jobName" : "PigLatin:exec.pig",
+ "hadoop.policy.file" : "hadoop-policy.xml",
+ "dfs.permissions" : "true",
+ "mapred.job.tracker.persist.jobstatus.hours" : "0",
+ "aggregate.warning" : "true",
+ "java.home" : "/usr/java/jdk1.6.0_24/jre",
+ "fs.file.impl" : "org.apache.hadoop.fs.LocalFileSystem",
+ "mapred.fairscheduler.assignmultiple" : "true",
+ "mapreduce.tasktracker.cache.local.numberdirectories" : "10000",
+ "dfs.block.size" : "536870912",
+ "dfs.https.address" : "0.0.0.0:50470",
+ "ipc.client.kill.max" : "10",
+ "mapred.healthChecker.script.timeout" : "45000",
+ "pig.inpLimits" : "eNpb85aBtbiIQTgrsSxRr7QkM0fPsagosdIns7ikovGS7MzjiXOZGRg9GViKM6tSKwoYGBiYylmAJBdQEx9YU05iXrqeT35eunX3kwln-pXvMzEwejGwliXmlKZWFDEIIBT5leYmpRa1rZkqyz3lQTcTA0NFwX8oKGSoY2CpAABKAzQW",
+ "mapred.tasktracker.map.tasks.maximum" : "9",
+ "mapred.job.reduce.markreset.buffer.percent" : "0.3",
+ "jobclient.completion.poll.interval" : "5000",
+ "mapred.job.tracker.persist.jobstatus.dir" : "/jobtracker/jobsInfo",
+ "pig.mapPlan" : "eNrdWl1sXEcVnl2vf7K2Gzt_pYQkTusWKdBdaJI6wZDGG9uwyca2bDehNoaO747Xt7l_uXfWue5DpEqlIBFAKK1EJJAQqoRQI6CFIp4gpQ8ECR7gCfFQoj6VBwISQfBSlXPO_dm767s_sdc8sFLGO2dmzpk535nzM9mbf2Odjs2mTLuU4RZXVkXGUkuZZa5cEkYxs8qLpmllhCuUslRNQxgl1YApq-uOqnCtwNeFnbE0bjiZGZ82Az3mfRJJllxg_cBoemVM0_KGVZYF1mlaOrcke6QAQrOe0CwIzaq6pWVBjJY9X9akep5bo67NHq3ZGs4ikZlpS9hcmjZKvPZL5zt3Cm-vJllPgaX1SdvUJ4ol4VxmV1kCZOrnxLoj2c7Cc3yN-0KAf4F16wXB1wSMDUTGCqojYTClT1vEoqPAuvRZ05TUSxXYA_qcuSJrxfQRdd6M0nr0aN-1QNuH446EcjPBwX0FJpPIIaUDhXZBqwdxm970z3FnFcY6u__81tv7nv1DB0tOsrRm8uIkV0AxebZDrtrCWTW1oms9dZp49l3pgXYA_qWA2TNbAd4WGsdBrgVQgBlMT5r2BPCr2MDAAhtUnQlua-vzwtZVg0tRXGA7LhnmFWNOfV7kWbdh5qXQnQX2gGWbinAc1SghrosstcxLgM7uRQ8eoJWy08vPCUWOwmCRS46q6S2wARGVANuSbFcE0pwq5wSC2qMa1nzZ0oRk-2tNENllaRAmplU0WNyFh_oi26M682ZOTMKxpTBEccy2-bpkycUFwN60aHBWOEL683vRTsG-VsCMJJterBXnazvraTtbo-1slbazwQULdA3H7wVwwWLm1y004ORiDszUAeY6l-wTsddLM0sVjt7U7Bz9gQN3yDwo7bHahZYwPBVOuBx4iFA_Er-c55eELdlQfVXSDLzKha0YW-3xK_bVt8A6uaKUdfA1BNmYlCijmGd7bHG5DLoRxRluc00TmuroOZau6A18A9dU7oTGQgY2J20wQDhjJzFEOPtBt9QJXEC_BpvkJTFvcwU18NG6eitEJwLTQdNW4YRwMlMhSw1YdptlGZGQtrgtDLJAyWa2ZDzko7NRH41422g241tiPEuaRHQPN3XUFcjIq4FLjrE2MlNclg2WwbxR8nzDTSXA1EjsOcuSahEwdBTTEqjTvW7gWfdLn8zYB_BxJUvMWhYK2V1xr3S9MRS4L_zxwI3f8O92sEQe3Ca4LGKUuJKCNg2Lltpp2cPTNeYx9o3fTw3-6f1vJ1kHiEezy7Muc2UFHE1gvv7pPvA_2D3kUtDA8J7byvY8hCNqzbE-W8iybcxJLssO3AvvOlVfId9Hu1bS03mKWahg3NIQdn3dYZ9R20Vtz4YwldhqmAKwbIwophENUzO2iRusnKsLgo_q-GSI2QbEqLS5JmwMqBiw9lZCU46XplfIuUHI2uMdfw7ovnOE4TzrcSS35RlTg3utmFpZN-Cy7Y1EpNC88PYst_uEExtolaMyF1W9hxT-EZ-8EzsHvevwRBVUzIMqMCdsh5PBRWLhtMpt8DIUtIIMBBRREvaud7_36r9f-MqJJF6gzjWulQWceaAyb6qsLwv7pZuvHOh9-c7XcIckwI3hfqH9tnAG_X-onRTEEQUoU6acEgKgL7DeFVVoxTk_tB6t9VlwRcyyrQhvwnDQnawswrxjpWwocxba3EO1DCb9IZxmC65hZAozUrpOuXUpRgm2x6tgG4jANhIBzdcXdjNuUyxBrZ-sUWvzM0U0lmMpScG0tygcxVYtBIB8Ejh6g-vC_x5mJ4caqxDcxghUCLwszRIkWjZmjEMrkG0PzailIdrDUIBGJ2SRwkbX_WDNEQKtVrJpyON3KBp3nKlgT4usB1PlMXtDkhnmADu9pAJmeCJx3aALFUxtxFsuq5C-GxnY4xzwhJAP18h16_i1JJKzBBA2n8Zm1I0DcLQhi0zIZ5T4uPE20NDHsi0NJ0Lhn3GZ1ZJLR_rHqgx5V7v8D_aPYt-NHRqpkrq7-fW50ML1wS-nR7A9I1kH3F2a8Nkm4H8RmyVsLnqC4sQvNWSxFPK5QHzc-BNsN_i0jS9tHvy97QY_2Qr4-yJS0_HgX24V_HQAPt76VsC3iT0269i4scg1ZnE55EOBYd2NP8F2g0_Cn988-B9qN_gdrYD_UPObf-1-b36ntQp5R0vwv4zNdWy-7omK28D1hiyuh3yuER83_gzbDT9t4xWA3y3b-BRy8ZtDJ_f8YundIIlLVWEV6BHmeu8hbSubRr__8X3L90q_eyMUHIalpdBMr9Meczf_-eB_unrm74RTR0bSI5BBZOIqzOibScZLYDJVGVCScsZ98opZEGtCG1OwSpgVl8uqjcljjz6GJZoIXhL79UgSFTwTdntEr_R3fcMKNN5FGu8JzjQOO_3U_e10eGPelozmbZEqsoDZr2EayGlqQ-rmJz9k8eOSJZ88RuZ-g9b-iMg3kDxCZCVCVpB8nMhXPTIZ6FUkP2k1NjQ69htx9n2DYP153NBVGvpJ3NA4Df0sbkiJeOJUhfxjbH6KzZvYvO5udBmAyrPtf07FRwBtVnAoFo2SqOTc-3L4OFUU7gLrVp0ztuk4-P7lnBPrZ0zdMstUv1KfatIFtlt15oRiGkVurwfUHOu-BF_BBhZYr553xqEcVQ1FLrA-3S-Focq382xv0BVFKGmd89zCl1soivWQZ9X8gxvoVQtzII2rxjlPdo4NONGd0XYGHUjALejiK_sFLBkX2M6yI6JngHujec-rwZOZPskveUW4_2g3uGHj_jX8cP0d-jM6rfDttxDZYaFK4gMhvfJSHFNePxwJN4ZkXWen81MT47FhZ79rxcavh31nugM7v66JRAkUMuIZR2Ak8RGyOvg-Uh0GNx18sXvc9XdeJ1Ic9yY1jjW122WtBKg6An-Lze3YuHi74cLbtLrRwu0JqAzD6EYEPFK6Crj-CHD4fFmwrDpmE30UvBMH5xae5jYaVKJN2VzEVhoncsnmidx7953I-W8K2J1oS6H_V9oHNn_H5m6sbd1tyOK9kM9d4uPGH3W78z0S_o_Np_upNhrIYisG0tnUQBKpTdf4_2qMe6KT2GPTi006DrSAWodFKuRDk3vd-BNsM-6e8L7N497dRtzXWsG9p2l5nzi46fK-Ge6HiD02j2IzHAvacEMWB0M-w8THjT_BduNOwh_bPO7pNuL-rVZw721-309sobJvhvxJEoDNU9icioXtVEMWJ0I-p4iPG3-G7UaehJ-myh5nvVq_mEfCDwLffjf0FmQ6p2j0h2GVjb3XGta39yJV4z0oD48_gYpPjFfICSw9j1MxmRiIkAeQfJTIRyo1ZuIIko-1UGMmPh-naaoWExfiDPIeDS3ErRqgoS_EDR2JrTETT2PzDDaL2Fx062R0v6oy96FoZUGlQWwO9M5mqgq2yari8P9xVZHQsdFi76TWcKFGqxst_N9WFShYxLxiXGz_K8bTBpAqjxcdeSjcuSPz-EjRbwhXztIvCiamZwosVQTWuKu-mDr68ai1R-wqWdElrb3ltmRzTvC_0PSDD-83Ysb7X7168thrhztYAn9-pkr6bdVZ16K3w7PuUOrNXa-Xrwdvh8HFYC5t91YcvC_WDRmU132Z5uCEA_QAgM1f6NLiwEp4qLe8k8VLaCTgxUCKx_tAwJukEG-Swm6RFDdGq97Cd7bTTDvCnb4Ua5neBurpmO13wcz_CyS-k-w",
+ "dfs.default.chunk.view.size" : "32768",
+ "mapred.reduce.slowstart.completed.maps" : "0.9",
+ "mapreduce.reduce.class" : "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce",
+ "pig.map.stores" : "eNpb85aBtbiIQSQrsSxRr7QkM0fPJzMvOzXFJ7O4hEczONYroUOJmYGhoqCchQEIKgCPiQ5e",
+ "pig.maxCombinedSplitSize" : "536870912",
+ "io.sort.mb" : "300",
+ "dfs.datanode.failed.volumes.tolerated" : "2",
+ "mapred.hosts" : "/etc/hadoop/conf/mapred.hosts",
+ "dfs.https.need.client.auth" : "false",
+ "hadoop.http.authentication.type" : "simple",
+ "pig.files.concatenation.threshold" : "100",
+ "mapreduce.inputformat.class" : "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat",
+ "dfs.datanode.data.dir.perm" : "700",
+ "ipc.server.listen.queue.size" : "128",
+ "debug" : "INFO",
+ "io.mapfile.bloom.size" : "1048576",
+ "fs.hsftp.impl" : "org.apache.hadoop.hdfs.HsftpFileSystem",
+ "sun.management.compiler" : "HotSpot 64-Bit Server Compiler",
+ "mapred.cache.files.timestamps" : "1364953685050",
+ "fs.inmemory.size.mb" : "200",
+ "dfs.datanode.dns.nameserver" : "default",
+ "pig.optimizer.rules" : "eNpb85aBtbiIQTArsSxRr7QkM0fPI7E4Izi1ZJdL69RpO7abMDMwVBSU8zAwMAjYOzCAAGMJg7xbZk5JapFPfnpmsmtFQVFqcXFmfl5wZm5BTmZaZmpRBQDnJBzz",
+ "pig.additional.jars" : "/home/billg/ws/git/ambrose/pig/target/ambrose-pig-0.1.1-SNAPSHOT-bin/ambrose-pig-0.1.1-SNAPSHOT/lib/guava-12.0.1.jar",
+ "mapred.child.java.opts" : "-Xmx3072M",
+ "dfs.replication.max" : "512",
+ "mapred.queue.default.state" : "RUNNING",
+ "map.sort.class" : "org.apache.hadoop.util.QuickSort",
+ "mapred.jobtracker.instrumentation" : "org.apache.hadoop.mapred.JobTrackerMetricsInst",
+ "dfs.hosts.exclude" : "/etc/hadoop/conf/dfs.hosts.exclude",
+ "hadoop.util.hash.type" : "murmur",
+ "topology.node.switch.mapping.impl" : "org.apache.hadoop.net.ScriptBasedMapping",
+ "dfs.block.access.key.update.interval" : "600",
+ "pig.spill.gc.activation.size" : "40000000",
+ "hadoop.root.logger" : "INFO,console",
+ "dfs.datanode.dns.interface" : "default",
+ "mapred.output.compression.type" : "RECORD",
+ "pig.logical.plan.signature" : "-156495157",
+ "mapred.reducer.new-api" : "true",
+ "mapred.skip.attempts.to.start.skipping" : "2",
+ "io.compression.codec.lzo.compression.level" : "7",
+ "java.vm.specification.vendor" : "Sun Microsystems Inc.",
+ "mapreduce.job.dir" : "hdfs://hadoop-dw-nn.smf1.twitter.com/user/billg/.staging/job_201301170859_1562015",
+ "io.map.index.skip" : "0",
+ "java.vm.name" : "Java HotSpot(TM) 64-Bit Server VM",
+ "hadoop.log.dir" : "/var/log/hadoop",
+ "mapred.cluster.max.map.memory.mb" : "-1",
+ "dfs.namenode.logging.level" : "info",
+ "fs.s3.maxRetries" : "4",
+ "java.io.tmpdir" : "/tmp",
+ "java.vendor.url" : "http://java.sun.com/",
+ "mapred.task.tracker.task-controller" : "org.apache.hadoop.mapred.DefaultTaskController",
+ "mapred.userlog.limit.kb" : "0",
+ "topology.script.file.name" : "/etc/hadoop/conf/rack-topology.sh",
+ "hadoop.http.authentication.simple.anonymous.allowed" : "true",
+ "brief" : "false",
+ "hadoop.rpc.socket.factory.class.default" : "org.apache.hadoop.net.StandardSocketFactory",
+ "mapred.input.dir.recursive" : "true",
+ "sun.boot.library.path" : "/usr/java/jdk1.6.0_24/jre/lib/amd64",
+ "fs.hftp.impl" : "org.apache.hadoop.hdfs.HftpFileSystem",
+ "pig.info.reducers.requested.parallel" : "-1",
+ "dfs.namenode.handler.count" : "64",
+ "fs.automatic.close" : "true",
+ "fs.kfs.impl" : "org.apache.hadoop.fs.kfs.KosmosFileSystem",
+ "mapreduce.job.submithostaddress" : "10.34.8.114",
+ "mapred.map.tasks" : "2",
+ "hadoop.proxyuser.httpfs.hosts" : "*",
+ "fs.hdfs.impl" : "org.apache.hadoop.hdfs.DistributedFileSystem",
+ "mapred.local.dir.minspacekill" : "0",
+ "pig.reducePlan" : "eNqVkb9OHDEQxuf29uA4_gSERJciitJ6H4ACKDhxiuEQpEqVya65XbDXxvbBXopI9LwAUvIGPAMSoqCGkooX4B2wvRfldIqQsoW1M9_4-82Mr1-gZTTsST0gqDDNGVHFgHzH9ISVGckxk1IRVrF0aAtZsnJQlK4kH5kiRU5xxDRRHEtD9se5fRdB_TUiiL7CojPqH21x3ivV0FJoSSVQWfhIHTSpoYmDJoVQPHEYnuwOuS12Ua1XGj5NtearApL0FdNopfbEyxvz65ne5RG0KXREV0uxnQ2YOYWf0HBM8ZmNjIV39BjPcAxx_hRmBWV4xpy2PKHRwlgnxqKvgkWTwow4kNKGKKawJA7lkZ3GLITsFzmZa4vJuFJu2x_-NZLnkj-DjxcYRd4hFi4Tugi3V3ybdfkOmtxprdmn27u1bw9NiLrQ4RKzLqZuMT2Ys7lmJpc8q9TGZv0o523v7H-r0FN7LCwEYbkWNKz-xWxpjSO_kuri8f3VPf5uQqMHsSl-sCq0eh67o_OGnRfmpytn3q7_L3lyKlCvNYnHWw",
+ "mapred.job.map.memory.mb" : "-1",
+ "mapred.jobtracker.completeuserjobs.maximum" : "20",
+ "mapreduce.jobtracker.split.metainfo.maxsize" : "10000000",
+ "java.runtime.name" : "Java(TM) SE Runtime Environment",
+ "pig.reduce.package" : "eNq1V01sG0UUHm8SJ3HSNmnTFqkKKiIFIZBdJMSPcoC4cYRTN7bqSAhblRjvju1tdnems7PJmgOiFziA4AISSEVCiGNPnDghoR640ANISIhT1RMXuIAEQkLw3uza3jhO2srpRtp4Zt7M-_m-ee_tzd_IhC_JFS5bWSqo2WZZYbeyDWpuMc_KtqnFuciykJmBsrnHvJbtgUi749smdUq0w2RWMofiInXKgkmquPSzlfI6t70KHENbjERPyiCZdTJttgNvq2q_yWpkumlLX23aLiuSWYf6Kk9bRc9iYY0cxWHRE4HaDIQDsmmP7VxknRo5EvhslTVp4KB4nYxZDV-RR-sl8CEX-ZADH3IWVTS3Ci-QWi6RySaXBVhUxBqUjL3NRd7mBrzN7fI2N8TbXKVcFsp2wSdrLdICCo_ECi8zH0xVZHUktdEpcOyYZODumaHe6lAth5K8fvh47sHySI1MWbavbM9USSgBIdsHpC5wV_DAs2oko8fauDyZ3IKfHQGIT3uBqxH2B0GtkWMwrjKTexaVHdhcJxO25zGpiFGvQRDgEEWOl67SbZpzqNfKlRtXmYnxycDSiq-VXSNvkfGS1lj0mlyRY9EGcN3JXaICWSG2WmiNIq2HwIo4Zkvxf1SE6JRGQacSj7p6-oDM1sgENc3ARQQwritKoQ6rSBYkuxYAb5hVoZI6DnNs382TjNSsQrtKsNex4c7timpVSdvD2zOhD4wDmtYDEJ1LxLMEVEDWO2Ay-iqpiXA9ORhWwbzuhoQg7Jzn0gZ_wU9u6lj6qG4WMOKBQn3xMCOoZJ6qgIGKVEZCTcAZgFM8hyfGFwxVjQFSjw0gZbvCyeKu7N7wp0pk3L2IvHxi0CjcppXluttAbjkUkHmX7qkBRPtKjHVi2Bbg4ZtcaILPhCJeLqh4mpD_4PEEPKDhBGKUxZBnV6SkHQQqvP7j4qff0c_GSKpIxn1IXNEpO-PwymjDNkYhqQ5sNhnYhAtATzio3FxxHH39wRsuXCoUeXxo4KLrCjy18c4CKufuGTPU-P43_o07pVttg0wBadw1yd2C1YqwfQR0IlZ6AMuTbonRbdal2LhbFt2ltHuZ8x75jrpV3lSDZ83q2U2enJtyk2Md0qFs0sB0vYujZMRsghlthd4938fxVeq3YW1i8pdvb51644cxYqyRjMOptUZN8B5Sq2oDi9vcsULx8itR5Hem8GT8GWqbFuOFWb0w1184mWDCA0qeOVj-gZaTBhMMQH4URkb1M8HCPJmVTAXSqyqqAh-AjpIhmjAZCkOQFCEvRBYEEsvOax-efWnh6yt3YULfFiOVEmIfj4weYJhHs0VPsRaTx-9-_uVf19990cBrN7FNnYABnef6chuB22DynZsfL858dOe9riIChy3uy50KtRNZyABm61Ks_QCvfF1FY692W5Xn3GHUu31Wvv3TjX9-B6tqXasE2cexFE6f168ItCxO6l_P4uu5-9jXzVhjv6ZwBf7QsMbh9yyDXVk_TOThNEl79MzVyLztF6h0OptMulDeoAZDt7Tl8R0Pm-AimfR4UTEXmqCjQnKT-T6UXExgdTLeoC0osyfqexsdWMSWDwMKNWuOJTWAWb0yrpNn3lZVhtV5Cip3sjXK6EqOyqIEVycLtr_J82wNvFPMY5YuGbiWhjTHhV6Eu8RULD-DeReSZxMypiLl-kj1eLC3AS9n-i2Kj81fHikNh7tUkfNDy4XDW_0TI9FcVf_D4g7fFEMKdK8pKYQUzmBRIw3xUfjjEt3CVubs_j23llgOMSYZHcnjMQXWcLCQLMiDeTOeesqILoVBdkkZkZQen76fHDrk5u0yaEkb5I_IfkgQEonKvV2fCJIjN_vsTwOnbT-ehnKoPwb4NpNYq_AenOwzHtr-clMHE27CQoR6FeZjMPDLkEz5ikp1gTtQsU3uBK4HnDiZIHqvwcE2oXHYHhb2zCUTigYfXxcjuENF0uvl4kZhNQloKgZ0xG_uwZuyVB5onlc-uL0x__O_nxhkDDo8bMqLJM2bTbi53Va_3z2iSdM4rAzSEYvPgOk4fD4Mh02Xw4Oyf1kH56CNh9k-JJeJOLwLdC66QPgu4cuM0dbhG4Y0ji8_aJB7pdUYHmnrwEjrcm8etPHhRToMcO3pqD_ak-2gk4pqxKExf_mLZ041_mx9_1WvLetxzdJ9W_7mH6f_Tk9t3ukJQAOCz__Yg6Kr",
+ "jobclient.progress.monitor.poll.interval" : "1000",
+ "pig.inpSignatures" : "eNpb85aBtbiIQTgrsSxRr7QkM0fPsagosdIns7ikovGS7MzjiXOZGRg9GViKM6tSKwoYGBiYylmAJFcJA5tPvKGuoTGQEQRimFQAAMlDFfw",
+ "sun.cpu.isalist" : "",
+ "pig.info.reducers.estimated.parallel" : "1",
+ "dfs.blockreport.initialDelay" : "0",
+ "mapred.min.split.size" : "0",
+ "user.home" : "/home/billg",
+ "hadoop.http.authentication.token.validity" : "36000",
+ "dfs.namenode.delegation.token.max-lifetime" : "604800000",
+ "fs.ftp.impl" : "org.apache.hadoop.fs.ftp.FTPFileSystem",
+ "dfs.secondary.http.address" : "0.0.0.0:50090",
+ "pig.exec.reducers.bytes.per.reducer" : "1000000000",
+ "mapred.output.compression.codec" : "org.apache.hadoop.io.compress.DefaultCodec",
+ "java.specification.name" : "Java Platform API Specification",
+ "mapred.cache.files" : "hdfs://hadoop-dw-nn.smf1.twitter.com/tmp/temp-977624468/tmp-278376779/guava-12.0.1.jar",
+ "mapred.cluster.max.reduce.memory.mb" : "-1",
+ "pig.cmd.args" : "-f /home/billg/ws/test_pig/exec.pig",
+ "mapred.cluster.reduce.memory.mb" : "-1",
+ "dfs.web.ugi" : "webuser,webgroup",
+ "mapred.task.profile" : "false",
+ "mapred.reduce.parallel.copies" : "30",
+ "dfs.heartbeat.interval" : "3",
+ "pig.streaming.task.output.dir" : "hdfs://hadoop-dw-nn.smf1.twitter.com/tmp/temp-977624468/tmp-1644509149",
+ "java.specification.vendor" : "Sun Microsystems Inc.",
+ "java.version" : "1.6.0_24",
+ "local.cache.size" : "10737418240",
+ "mapred.child.ulimit" : "6291456",
+ "io.sort.factor" : "30",
+ "hcat.pig.inner.field.name" : "FIELDNAME_tuple",
+ "mapreduce.map.class" : "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce$Map",
+ "mapred.input.dir" : "hdfs://hadoop-dw-nn.smf1.twitter.com/user/billg/ws/test_data/pig/data1.txt",
+ "pig.pigContext" : "eNrlXFlzG1d2boqite-yVsui5U22hQZALCQ1qcxQCy0qpESLlOMoqVIa6AbQZG_uvk2CmsxU5Q_kNZOH_IP8gFTlZZJMVV6TPGZe85pfkIfkO-fe3oAGRcoVvUQzJom--7ln-c7S-Lv_0maiUPvYD_u6ERjdgaUHdl-33cDR1-3-Q98T1lBo8t_UEe38inbOtHpG7Ih1IzQcx3JeaSds7_EwcAzbe6Wdsb0Vx4kjERrCD1e1i4ZjG5FlbnRDOxDLtmNFQju3umXsGNVY2E51zQh-tppOuur3V60dyxHatVXsqSr3VHX8fnOryi2qs-1Z5kPfdQ3PjH7Ufq1NrWrn1ePl2OsK2_eS58etodXd3AssoV3PT4qDVh-rJsx60jEi8Zq3K7SLcoeO4fWrGyK0vT6ty9tYD_3ACoVNB7mSO0j2nCYLQB1XHfd8rteqHQm0f8DtvMMPqHc6lp7MrGqnooxeSSf56KkRJk_OR9t2EIC2AztYN8RAPh8G_4N_moZrvUrr6rSuvmp725b5xIgGIHjzt8_-7Lzz2_8-ok290k4Z3a4VRc9D0wqHoXYhG6N6zxz7j3_85yt__m_T2pFlUMk3zGWji8td0U6IQWhFA98xh8HPf8E8cnr3OH6epz-HtIUrOc5i8ul8i7Xo2O3bR37Xnka3YPespj2bxYgPtKMrz5afD_kgJ0pmnNjwaxx2hIeTq1XMq11Et1A7y8eje9Ufe7GbbwyEdmJtaf3F40cvHz4W2tF6pd7GCS5nBMmuePHiv74J2n_4myPMYIp55eVhkUtFGgqj41iX_uHc5588_c-_PxAVj_PRTpHMCe2mpBt-9q1QL55SKIoFUWHVpTA09ojVhn_577f-5l-Mv53Wpla0o5H9xhoGtMDuUfw4Kel5ffTBZbmLIz-nXRwBK039qdDODAzT9wPdNnUIt9BmOrbjYP0rW35HtzxTDy0R7uk2FEa4Y0CAZxo1_BPah7avd_aEFemgnY59d7ej2BXadKs-J7SvXCMILVOnWaAzutvog5lsPMOjSO_ySWnjdCM84bnY7JGG8kOhOzij0P6667u62LUF1sZvuVFQ5_6E5zqRf2JjhFuxCq2WYwUDwxMdOzTLx4_14Elwd-p4woi2dUh5DwKN85lxlxTDdK0CEnwu--ARUSEhQiSMPtSOHvq-0E2bKF6NI3Cf9nGOZKGFZ_rWjquDmXkVTHukUgfZaRek-dRyYaS7BlT59OLiotBum71I7zh-d1uX4q8Lf9vyQM8eiO9i59NtonVyPWp_vIAeBVY3dgxh71i8QkzaFvvrGU6EkV8rThkIEehGLAaWJ-yuQZ10nKxjhX6EP_YgFkK78dkv6VT6wHetX1XVyLSRdgmuce0oIoWuRzE-9UM_hqyezH-4BSaLrB-ZvBEYg7mo64emY7u4Ie0YsQ5zz3Fejgl6o0qrVpmRq7tRtW-LquF2sD2c4hxLEwi7A9Lx-Y7XF_V6pVPDlc3mrzW5MXle04Qw4xLO1HT-3_1WrdbGuvfoLJ7hWp5vWroJhulLkkjCh5Zn7VYy6Tm-0G6qHX-lqLLrh9sGjgtZ82gctIZhRkbP0vuWCHZj28zu4AwWg4HBTzLlQvsypzbUdGhb8dYs1w_3yMxs7EXCglheUEeL-LMk1IfqYqqyrRqpvnfKyBBaLJkpIU7V5-aZFPX7OMzdjNnVr2gQ93q4tq7veVZX6MR-fow7-6C-IAlwMYo9HTwTGKI70B2JD47F3rbn7-JeKiUqZGCTAO5hUhzfEmgFrxuSTx8NcBX3q-pMFXO34nl65PbqqTRj1P2F2lxt5Nhq0qrpeyDxLbXswMJ9DR6SXsPQ7AbT7d9IOInkxu4lokC8ABI-RePs96BybDiza7ggQJjZjXxPoVULGiI9FJSj17WcMWYC0Y-KMMbsF3DJIEk0yOtliMICtvVZctEAEVANLKo--nVDUlRdrBJ7Ip2pchChDgCUunZAq1x7srm5XiWiOwM_Er9Yff5wafXJ843NbOGeYYcRGNKMHRAuDkwDR8r2ebTF1DuveoMwDjjNhbDP6FX-fcyPFBVnAG9iKLeFHEMmrEB_u8zmAIsGmVDX92xcJLRrZWy56ySm2InBYjpZpOdb0rKRssHFQoWQoupaTLKaTgT-MseXCZPbXhALvYMP2Ed-wDwZNlobSlr3O1uQBKw4hafSrO_AxPqgUuhgdJ-UArYGJuZGkg8wRbXbtysd26uih5JCPMm0VXrTtAZ_kDpSLfTVRNFEf5iF7gA35sQhWa4jdQy4pXgiggkIbbHHfAG6vjGKBoHuyQi7AzwxXLPdhHYjDnu9ZUC3aKA5mSrPF5lskFmHQkTzct7KKuUs4Z3bsUzTMpfkMzgrQGd9uqhnuZlW04lm1WbpNhLtwKYgUwxHPRbsa2U2G89Sg32H7onoD4ME7sEcMFhs8EG4yI9xq0I7G0VORTUPXZIIGoVrcZJDZrw31cj4nEWG1e6UXiVGrJp2tF1PdBA338sa5iY1NCY1NCc1tCY1tCc1zE9qWJjUsDipoV6b2DLx7PXi4TN75AIXWAlzP7B6uJmEOaT-I0k_zbfPWi7cA0e_3AAsKjMlE3VAg5R7op6gk6kHyxfMVxBnujMBgbQgUDzhW_jKgDxxSKI018xEFF1MmHNwrN31A_Zi4Sky7FOTfVmi4groDMJquwSwiavOKa6nZYmPAZuyJ4CSoxIMEJFAbqvLxGo2WS0SxxuhDAyA9uTRyI7pxj5KKEHeP2lZUjSsd3W3o0Dp6US7SlzxT3kGR1OBr4ufGyOfmyOfWyOf2yOf50c-L4x8Xhz5DH4ceTC6wzpvEeiEVTCrZ7o8HAwPvwBWD1k5V7fM7bre1muv55rVrdCqOnanmvRW9iaB4EA-UDgw_RJwHmu0FQS8NKo7XCgPbQoUvVUOLlLQOl3XaxIHsOcV-BgoiX9JHiVDQZGH_tcHmEHGgADHYD1jQA1lZafA0qcwU9TwFKyslcJKtEvn4Bn_2mjk8WXOyuRZGCLj9zqGx3gqFB3LEJkB-SQTy4zl4SDBwJGQ75B4JH2vJEZiFwwNoG0kjH9eNbymR1sREedUzrTCnG_E3uya3UUn3ms0u-J1dayemh0AGV1GYfTIsSyWG98j-3mEIP6NdOu0LSJ3Hg42W-ouMwGU7gr7tseVg4IOZ-Vji0JEgjY2VU2dxBHQBCjsCZJtyyOJzDkAp6Wbmhz-mPoIdiVWguskQXoQ4PAZXCQgoQIksYdVsPjpjZfPXm9sLj17tPTiUSbltgfZVoo2jV-kLvondgAA6djYXYLrmWuhmkxHOZgzcwrdF3lTkuNEq9FemK8tUojgo9RXVaAmIeoayJ-XkivMe_JqNrHGRno7hFZkYAvSFxpQSfAlgEd-f5_lFA_bzfTPRCRqOjwA-iy5ucoYs8IIppLr3oZBtbpGh6X9_2y-ZBJJ1ErXHDTiuYqoT54Q2p0Ip25B-vpSJihSw-FLshCflckWxV_IBub8UTYEkoFLYgJSGTHTRT86uuBo61HSJehPXEX4T2p-F8DaIYnBDi-qqX6MrVhqGcx0TIXToIxyXr1jvNkzrcTMpix7PsNiUgYyEbhdKjJ4ZLmBAnxyDg4Q05Yp-HVRaJcTzw7yjmmHsGgmun1eBQZOXEJwVq9a3u8s8VZBfu8rbJ_T4JL4H-bkRHQD8jkcYy87wlUiHYgQe4AD8EcsD7_gtwCHvJSPVoGM6dBXmfRSN9lez880gjrkKaUBpD64U90xQgqrcxQc_72uN9rNRZK6-sJiQ6oKnhMnFAAG5NqR8htYrpHOmXCC0XWicRX0SQlSMT3psklgnL_rMxKMG14_hpMG_rDIGiWeS9xxbVGkHws1uWAJT5CGIZIQbvnduGHK9dS_fWMHD6nrvf37PZK7O0jXB6_sYE52JJdF9cjvzXnj66tv_IP0kZsDgbECc7_yGKVynGm3cFEAGQncBdLaNzKnrusa8wj7qjRRXm8fayXWR8KZrmPgTCmKgHNMMOJqbkHyISHdJEsk_TkMm_d3pYHIO7rttgQRCZoEL5aCiCdGmIcNH5f5mlkMAvY1sjkqop2XVkBRjKHO5c9-qYYLN6BHv6pGkL6Py5iHr4CPL7RPD3CTORU0IVKQCwUop5v15r7hlnsljgj-ThxXJ4wLkfGpFmE8YpREQ-jBdp-IwvqDvFEKEQQx4VTbIHI5SnE0JhgJdoutZd9x_F3M94CbMsuxoOxyHpRCJ0a2aYWrvpHpnVsTIs2j2vpGMcYFlVLZsa3dCtklbWoWop4jSWYv8irkGJqe8bML63Z_FZvy7nMEnBMlF9VFBj42vKdw0QVlVtVDdtTPjUSd06N8XXIpAYkIYCElK4Qh4ihx6yiicsHow-nsU1xr1wg9kJEszF8B3rCQUfQZOm6iqwAGLDVh4E2777k4s80cr7bXKEfWXZV7o7CCF7sdK4QAWJR64ptM3WEcmjlICuVnpUK5SrPkxfJsdrvjwO1CZpvLJKNJknEpZwG3STNxkoJ1-6elAVZl49Io8UxTBu4-K7E2mWLMnOJFKBxcVgpOWWkQ94IFnPHw490D3bnUMtUMQ9Gf0QrsMLx_9nkljwLoxt62Toyd6PLG3Hx7IdPlSeQNUicdnCyCnYSg9EWlynGoh77boZz3BqRQbIzdwMkkQkke-HSDDvR5IcBJET1MveM7MYwyRBP8BXaF-E5h-Ok8wIEo7wuAVLpG3rdncSSAaUyaOofK9gkkS9iYavJP6JDEkxGpFlJHnuqXGa7pOofYC4dKAgwsxOgyT10-IkZT8TgZXlRaRN7DdH0OtzBjWhxd5aSqRBc4orTAjg9jIDsfq9eaC635tsyzDKIejiLF5otxsaFsg_6E-uQFh5GdDEi7CSOimdJ7T3yxEfhiFij-gS1mN3jPsw_TDpcoq-Op0Aobcr7dOTrlzSIhJqKtS0RaCjC5GA_EH3Ohwl9Yz4LOQst4IDr2ynebS2G0MXwRzn-3vVbrra_MP26uvLHrr56aq-3FF8HzeaOz9mht9_sHGy8X3iztri1t_cnzNw-Xlr5de9qfH3ZebbeeGsHyes8L3MgVy999v_xj94dlt-e0dl-5rQeb7ivjlRu8ePDdI-_pgzdv1KagKWy6ZGirLYMUqZicpWPwCikBXkweVfAInlZdr1c2ni2tbzx5vskh8cnN7EEBdO4Ylfocpal0jktfKeQdWF_7DHVOVH5wh43a_NxaWVyGU6yc2b5R8G0SBUAKgzjoxctnz1aefQsNSkqKZVSBjtlxFuKc_nex3d3eQMfSeIgNOxXGxEoKD-nj06hRT_3Ophy1Rua8Cz1FSXSprYuuzJ0xiS_p9GF-lwNKNik5duMQ_8d2hQ_l6vf3dObKaNemPB52ExBQkYJzd3y7niV0WTb0gCqI1mR_aPoxRLFt7Y3nj2QC-6MM8_a76A9oLG9KhV2aadglwQeccJfFFtCApAjuEbDxSSPdGBMvXq9ndAtA5IbytWyyuQF8Gp0MNiwF2fSTlXqLXK16az7L5cuIcQFkKiK-ePzwOQVevshDyAR4Q2Hr0kxwKRCThxT315ORbR7GykzqFPbx0aQw4v6hsQtSQYIKpiVTShL4nE6mkxDtE05yKr12d3PtqxHd9j1E6WwuUs3m9FzqosqWzMHIIszD0iiz8gDQ_CLFq011eWkGnm6Y2E8R4ajNxvrTsow2faiAB0QIiEBcMVm2lMO4iREPcwNk1RGIBf-DD3dUZjDPjeT2ACLK8nqZ55WkELisQd_uSHJfTwVMoSO2WIr2YyJMp6pkI2CbG_vYZGmMdXC8t-f6wDoGeQS5wMJMByTuZQb-y0SOyN5CSCHFPa502lNeZSom30yQeYiXaYTmBg-WVVJ7mT6VKVQy7yH5ghF83XQvHxLFOizAhTjfp_uH41VWkiDwILPln0-y5SOmvMhXA2zeSRJLSZiLYvxw91yQlNAuF5gkcRm0bad1Gl-Vwm5q_yM_cv0ov-75Ufdfyn6SlYVXOdyTtTW41kRto8_XWe1CmmykhEKEZS3C4EqGaYiZ7uzeBGo8ApYK7U4M0Jjf3LUcbi6T0a_HTViCdGnPXPuVgvYjc2n8NgRVKU0imfs2KRZWKBuPZ1_IptnH3o4d-p7LEYdvyourCC5jT5SH65EKG45F4e9mTkKgcodJ4cCYq6Dq0hIf244MWZemlQavAW8M55EM9BGhE8cWV6A2ppx6tJ1Ii6PAKTkglFUFleJo9rCxNxtYisKJHCIH0fevQQIZKlkB2Ik2QK4iBjHpviBXtS9vro9JhsyRkCxOjpEskrs3XrCWlQ2qR7CeyQ3Rtm5Ptp5dGZv5-uDxPaHKZcuqdJjPZtcdQ_T80J1dWl8ZLdH5tMQ2Kb4bS4LSQbuuqQO4UpCp0psdgbg4tXhN2DaLXtwemX_C3KeI4rtWR4-p8OM8_iIGuoffqlDuUklxQ6a7bxW90ECVeINgAZvRI5TyvsIYME11FcsYrpfQ8G0oQqGFrM5O6WccQCqofKjrVL0235hv1hfmmrlchETpcVLt155brDdb8M_OJg6wNEHqBKNJzp5tYbS86nPLK49XHz1bWnv8mnOfIyTLqtOSrNPdQu1nwUh1DFNVIaiE7cKkRF7g-w6trwrB9xLJl3u6lIoGBq-D8Ev9fmpA-DqoDNDlDMEQnq2kY1KAO1symvbzgi45y0Vcy0ViOEuXy-otqErezIUuy1ZcHslsJiaQgjk3s2CO5BuCzko3kMfEiU_H9oqJz5NJ9dzk7HY7i_jmwlDH2_P12sICW_UEi0qEd9K1h5Y5S8TKpDYr2ihTIgcLAxfKhlTQjbClMssfTzLLLJhURyA_V7hDhXpExULTfA6l1O24lKIfCbQY-0w93R_7JBVLEbm999_SVxygEzYRRgY5PAfovBVR-fLbu3UP0qs7oFIZcZCT4Pop5EFtkmL7D2F6kgJk3U2-V1KSDRgPeH7CzPDVjck-c2IRE2Eli594kypkfBaOjs6h3R9jK9xT8eKbhZLvUs1_DELRYWCZiCNrjbRWh2QxXydxM2X9YeroZPV95C-V12kS_FdiKG3Hp2PnpQGVbAQH1Wv7uhjKOSZ9EFpCTayXlIZXaHyFxlfSURU5SgXu6d2ArNa1rfBLUg3Bt3Y6n_ws1N-P1D1SgXVJPXJ9QpVpJEsPaEClw1VntEmJqFU4-VYJ1TsOfhNgVHS_miFPpZNAD767D5aXVlYfP5J-N0gDHKmKLTG0VQp1kl5Y549Dm99E2cgPTOpLjV0hc7yMvWFVSJfIpO_6xrpseEoN-Ur7FEex6k1joyPoLIe7iUobCWOUAkk1ZhkslOs48pJJLu9IFcJcteLFuez4zMvN5coCZfjG6lAUMlNxgSMEIr6ZBIuzuFJ5Zf6nbw2zkBU7sfrqef2H1_zexUdAYqnuJvruUNDK6Ko7TIRzdr_Uh3Rpb8rKEMJH1YginLIvV9II9dJS4iyl9vJ0otUqndp8uXXhUI40X6xbyHifziqgKPwIOED1zVeLhTsWfI0ozbeOFZz9Kq0dygmahEkKtbEau5bfCysL9Z4D2lu1pEyATvWGi3WnX24-lPVtVJU2AQLMlpV1lvjpV7apqEvlRNiO9-QbfGlxWbnDkQPQo2lqtu5JrFNo7XGOT7tuDCzH4VDnS88efsuwIA16JoWcpAwlkWffVhpUKFROUmwTCpVVM6vqu-Nymy-aztP4WAMwe54yJxxnhTEkgpVAqGkIBSSVOg06OCIAUsQq7fITOvED24OD-ND3KE1Bj--UJMzTqvUEMZHwn5HVl0OhCi__4C2Fl0ORM_O4h22jDxigmrCdQoC31Fetj70lQBsbdm1rhyHb0WZtsT0aKt4vC9oCKJUIdr_UK5qLhZS3c2hcGioWFPIkclUB53LlHKw4Lvx-1F7nCAIHF-zO5BCYRyEp7k-3lutOXahsqKbX6yBxe6FKOkjXDzbje8jobFHZUqWNhrnWe1uUrgPQpWIEdmVOb1XmarWF-ly9_p5PTZmY93109RjM8N6WHE3avZ8rdnrNLb5gUi7N97yujMa_56UnN7_3LcC0uL5XuouX5DEXtxFBKXtdqwrN5xE2qVJGuVryRlElvzzjM5V_LhxOt3f25tJJGSMVJuv65Dv5IUxKdRAaO5YnF5R_U_2v3tiXdKa_69GrxZFSt_Innf-N7xM2scJKQ29gHhosaTFK38kUOtBlZBrfBLA8gL7Hw_5ex_C2lfR_017InOmJo7b2xAC7jDjV4wDJkbpU53rretlKFVB-ADCWvNv1k0fL4dHAAF5g-6WwFWH6_OyFIu3qmvXQ6OxjKxmk81MsvljTRV1-yk1Z0qVw72_vytrw7d2C0Bd-J-5Vtgrd_x8eeb-q-uQ5-fBUtw_WTBsPNgNrLXpCaWC7A5Vd6zQOMZQinmIrFKzrW4ceyGDrsGOl7oioEUPn3mEgoX0MbR5-qGlYpLfqmTE_zGjLqaTI4zDjCJerDo13WVdVEfDizXcfLuHEO00BJ5gJfqjNg6vpuIcYk7SRXwJnNQvv_QTxUG2F-OJPmS760TFJ0BaIhw7BDFtwAyPwHtPFiIiVWofh_mQ8efk4wTvNENHIpAiw0sLw-uHHq3jP4YdbImoIkL59GJbIeVhtKmqITSs0Dj9B4jORn4gz_OT5MufnHWcBLzIoah6GgrFni0rzMMp2uxcxJDzEKvxFQKyZ64dYyPW727bwK4bjsHAcYkE_9CvkXi0cfMj-LvC7zEB3WT-EZsx7aE29cdiBTOT63GEHD10HbgFutHUYtokCHLGe_H63gZJOqmz1evH1V35BNFIJlUvJG7DqqXwJ9mzx1Zi0Ni1J_3BSVgZqqfjBEPyE9A06qJeQ0-_tSb7dh6pcKDxK388mtBn-LLSbo3Vq8n0EVQhBM10rFljGFKekEJdJsess8U_VNrmqepkl_-rA3-ui6qQ7hkNfoKLTa8e7tikG61a4QdHK4836YrNRo9zAST_3GtSZOWjIxmJFzOuW08qqN8fLqLgoU0Yo74yHU6lKmPNFgYytgkLq5eDruTo_aiFEncTHj7XmmnMLC7kvHcgFRouvYl0cD3hjIxlhtkQpYZKQcS4XQnco34PJpdW4iOJi0jMf-T_32S_TxzLofzlNL_VDIxjY3cjydkA6fikUT3-o179VDYXCrZtjtfdWGPqhHnL99ExNr9VaSWElf0vT_jUmd0pTHSM109WJ-Sgq6Cwrmi6b1hoGduGb0NJSEc4eZxH8W_v6qkK7khSvqBqS_Gt0dPhiMjmt19CDLmcrMV-9l1bAveUNFq6MziWPT-evUWiPR78swPPuVV1P5J50vEoHs8fBaEPkJS2FFFDylhc2b_f2sve8LhdTR9iq7Zv8ntJC_oWcSW_1lnwD3VWiIyUOki-fk6yUS9vcyV6HiKhEs-SlE1CUO_9sLLRfTCRRjc-9yULEqX6W-1zdFMXpE0ZKawtujb-0qL7thCVdO1N43y_LeV5ISmnla6Bgr9Gv6Mi-R4prAqJEwUzXSTUWVTAlGUrTINT1i_GyhPIvOJrm10fPRdD_OhUOyDoDta9hUP41hNmDI9mDD2QPslKUSNWOswzxn2dzQQB-MMOJUaGdkBEO_vvcSNp0-L_EiBqM",
+ "mapred.task.timeout" : "600000",
+ "pig.job.feature" : "HASH_JOIN",
+ "mapred.fairscheduler.poolnameproperty" : "user.name",
+ "elephantbird.mapred.input.bad.record.min" : "8",
+ "pig.reduce.key.type" : "55",
+ "pig.exec.mapPartAgg" : "true",
+ "dfs.safemode.extension" : "30000",
+ "pig.exec.mapPartAgg.minReduction" : "3",
+ "ipc.client.idlethreshold" : "8000",
+ "ipc.server.tcpnodelay" : "false",
+ "hadoop.logfile.count" : "10",
+ "mapred.output.dir" : "hdfs://hadoop-dw-nn.smf1.twitter.com/tmp/temp-977624468/tmp-1644509149",
+ "mapred.heartbeats.in.second" : "500",
+ "line.separator" : "\n",
+ "java.specification.version" : "1.6",
+ "fs.s3.block.size" : "67108864",
+ "java.vm.info" : "mixed mode",
+ "mapred.map.output.compression.codec" : "com.hadoop.compression.lzo.LzoCodec",
+ "mapred.task.cache.levels" : "2",
+ "hadoop.proxyuser.httpfs.groups" : "httpfs-proxy-users",
+ "mapred.tasktracker.dns.interface" : "default",
+ "dfs.hosts" : "/etc/hadoop/conf/dfs.hosts",
+ "pig.temp.dir" : "/tmp/",
+ "sun.boot.class.path" : "/usr/java/jdk1.6.0_24/jre/lib/resources.jar:/usr/java/jdk1.6.0_24/jre/lib/rt.jar:/usr/java/jdk1.6.0_24/jre/lib/sunrsasign.jar:/usr/java/jdk1.6.0_24/jre/lib/jsse.jar:/usr/java/jdk1.6.0_24/jre/lib/jce.jar:/usr/java/jdk1.6.0_24/jre/lib/charsets.jar:/usr/java/jdk1.6.0_24/jre/lib/modules/jdk.boot.jar:/usr/java/jdk1.6.0_24/jre/classes",
+ "mapred.output.key.class" : "org.apache.pig.impl.io.NullableText",
+ "dfs.safemode.min.datanodes" : "0",
+ "opt.multiquery" : "true",
+ "mapred.job.reduce.memory.mb" : "-1",
+ "verbose" : "false",
+ "pig.disable.counter" : "false",
+ "mapred.max.tracker.failures" : "4",
+ "mapred.fairscheduler.allocation.file" : "/etc/hadoop/conf/fair-scheduler.xml",
+ "hadoop.http.authentication.signature.secret.file" : "${user.home}/hadoop-http-auth-signature-secret",
+ "dfs.df.interval" : "60000",
+ "pig.log.dir" : "/var/log/pig",
+ "mapreduce.reduce.shuffle.read.timeout" : "180000",
+ "mapred.tasktracker.tasks.sleeptime-before-sigkill" : "5000",
+ "mapred.max.tracker.blacklists" : "4",
+ "jobclient.output.filter" : "FAILED",
+ "io.serializations" : "org.apache.hadoop.io.serializer.WritableSerialization",
+ "java.awt.printerjob" : "sun.print.PSPrinterJob",
+ "mapred.jobtracker.taskScheduler" : "org.apache.hadoop.mapred.FairScheduler",
+ "io.seqfile.compress.blocksize" : "1000000",
+ "job.end.retry.attempts" : "0",
+ "sun.jnu.encoding" : "UTF-8",
+ "ipc.client.connect.max.retries" : "40",
+ "dfs.namenode.delegation.key.update-interval" : "86400000",
+ "pig.inpTargets" : "eNpb85aBtbiIQTgrsSxRr7QkM0fPsagosdIns7ikovGS7MzjiXOZGRg9GViKM6tSKwoYGBiYylmAJFdxIUMdAwgwQvlFDCr5Rel6iQWJyRmpegWZ6XqZuQU5egU5iXl6_gWpRYkl-UXeqZUMEMDIxMDkxcCUmeIDtD85vyC1hEHIB-QIfaD6dP3gkqLMvHRrsIUgwFsCVVaBYS-IzwxVJg3isFVUAABL7zY-",
+ "io.compression.codec.lzo.compressor" : "LZO1X_999",
+ "webinterface.private.actions" : "false",
+ "mapred.healthChecker.script.path" : "/usr/local/sbin/healthcheck",
+ "java.runtime.version" : "1.6.0_24-b07",
+ "mapred.tasktracker.indexcache.mb" : "10",
+ "ambrose.port" : "8080",
+ "fs.checkpoint.edits.dir" : "${fs.checkpoint.dir}",
+ "mapreduce.reduce.input.limit" : "-1",
+ "mapred.output.value.class" : "org.apache.pig.impl.io.NullableTuple",
+ "mapred.mapper.new-api" : "true",
+ "tasktracker.http.threads" : "50",
+ "user.timezone" : "UTC",
+ "pig.client.sys.props" : "eNrlXNlyG1d6hrbRYi2WJWu1bFqSbdlWN9DYCGpSmaEWWlJIiRYpx1FVStVAN4Ame1Mv3CaT10geIg-Qqtyl8gC5n9vc5hmS7__POb0ADYqUK7qJZ2wQffb__Mv3L41_-e_aiTiqXd4wt0w9TRxXX42C0I4Sx44XPvvPvbD71_98tHZkuXbKsodm6iZxUruyTL3r1Lue9_79TlS7lE_z1IzHidl37Uv_duGbr5__178eqx1dqp1xA9NaMgdJED2rnU7GkR2PA9faCf_wxxr-OfqH7VP47__Uakf-PqmdG5tWEIS6Y-lxEiW1E33HdUdYfyPo67Zv6ZGdRLu64yd2tGW66NBq4J-k9rkT6P3dxI517E0fjO3BZpx6Se1Yx2gmte89M4xsS6dZksgcbKIPZnLwDI9ifWBihB47e3ZSO27whBdSa6g7XhhEie46cZLU_mkQeHqy7SRYG59io6EzejDjuU4Hn9kYgx52qdV27XBs-knfiazq8VM9eJKkdlMeLzHjTT2MgqHj2jiflQ5s3N2xhgYSfCP64BFRQREhTsyR44_0KAgS3XKI4vU0xu3WviyQLLLxTN_Y8nQ_9XgVTHtUM0B22oW9Yw_kclGse-YO1lxYWEhqX1nDWO-7wWBTNwfYS6wnwabtg55DEN_Dzo91idbqeuT-eAE9Du1B6pqJs2XzCmCxwMf-hqYbY-QPklPGSRLqZpqMbT9xBiZ10nGyvh0FMf7YBUMmtRt3_0Sn0seBZ_-5LkdmjbRLcI3nxDFGY-UU30ZRkIZJ7Uzxyy0wWWy_Y_LGYAzmokEQWa7j4YZqJ4l1mHtO8XJM0Bt1WrXOjFzfjusjJ6mbXh_bwykusPSAsFsgHZ_vlLGgG1q_gSubK16rujFxXsuCGOESzjV0_t-DTqPRxbr36Sy-6dl-YNm6BYYZCZIIwke2b29rufSc6nXbcsffS6psB9GmieNC1nwaB3k1rdgc2vrITsLt1LHyOziHxSLTw38hKJjuuyAa6WbIwiSnQ9szf8X2gmh3CWRb240TG2J5UR4t5u-CUJ_Li6mLtnos-96uIkNks2RmhPjEaM4zKYwHOMy9nNnlRzxOh0Nc2yDwfXuQ6MR-QYo7-53REwT4LE59HTwTmslgrLv2lo0TnUz9TT_Yxr1oFSpk7JAA7mJSHN9O0ApeNwWfPh7jKh7U5Zk0a1vzfT32hkYmzRj1oNdoNiaOLSetW4EPEt-Sy45t3Nf4Eek1DM1vMNv-DcVJJDfOUIkC8QJI-ByNc7-Ayqnpzq3gghzfnlsr9kxq9ZKGyA4F5egPbHeKmUD040mUYvaLuGSQJB4X9TJEoYdt3VUXvemEUA0sqgH6DSJSVAOskvpJNpN2EKEOIwx3Qlrl2tP19dU6Ed0dB3Hyx-WXjxaXn75cW88XHppOFIMhrdQF4dLQMnGkfJ_HO0y9T2VvEMYFp3kQ9hN6nT9PBrGk4ollx0-h3HoFhlSsQH97zOae6ZsjPPEC38FFQrtqU8tdJzHFTkwW09kiPd8Rlo2UDS4WKoQU1cBmkjV0IvB3Bb5UTO74YZrofXzBPooD5smw0dpQ0nrQ34AkYMUjeCrM-BZMbAAqRS5Gj0gpYGtgYm4k-QBT1AcjR-s7fh09pBTiSa6tspumNfiL0JFyoe9niib6wywMxrgxN43Ich01MOCW5IkYJiBykl3mC9B1zywbBLonMxqM8cT0rG4b2o047O2GCd1SA83JVPlBkssGmXUoRDQvFa2sVM5sYJ94fduybGtRPFt1RkA_I7qoF4WZlrOJ5uRm6TaUdmBTkCuG4z4L9rUqm41nmcG-TfdE9IdBAvdgDhgsNvggXBykuNWkdj6OXU0273gkETQK1-KqQ-a8d6SV8zmLDKvdI3qdGLFuOfGmoXQQN9_PG5qzGlqzGtqzGjqzGrqzGuZnNfRmNSzMajAaM1tmnt0oHz63Rx5wga2Y-6E9xM0o5hD6jyT9LN8-a7loFxz9eg2wqMqUzNQBLVLuSj1BJ1MPli-YrzDNdacCgbSgG4wI35qOD8iTRiRKzXYuouhiwZyDY51BEBKbxKEZMeyTk31XoeJK6AzC6ngEsImrLkiup2WJjwGb8ieAkpMSDBChILc9YGK126wWiePNiNEEmDMmX0J0zDb2haKEm8YkrKxoWO_qXl-C0rNKuwpc8e9FBkdTia_L31sT39sT3zsT37sT3-cnvvcmvi9MfAc_TjyY3KHBWwQ6YRXM6pkuDwfDw2-B1SNWzvUNa9PQu3rjbbNd34jsuuv066q3tDcKggP5QOHA9AvAebLVlRDw0qTu8KA8akdA0VvV4CIDrccMvSFwAHteYYCBgviXxFFyFBT76H99jBlYwTqAY7CeKaCGtLJHwNKfYKa45UtY2aiElWgXzsEL_lhrFfFlwcoUWRgiEwz7ps94Kkr6tpnkBuTrXCxzloeDBANHQr5F4qH6XlFGYhsMDaBtKsb_VDa8pUcbMRHnk4JphTlfS_25FWeATrzXeO6ZP9CxemZ2AGT0eBA5IT5c22a5CXyyn0cJ4t_Itk7bInIX4WC7I-8yF0DhrrBve0o6KOhwXjy2IfhmQhs7Us-cxAnQBCjsJyTbtk8SWXAAzgo3VR3-pPwKdiVWguskQHoY4vA5XCQgwTRxzdTHKlj87NrrF2_X1hdfPF589TiXcseHbEtFm0UOMhf9aycEgHQd7E7heuZaqCbLlQ7miaZE92XeFOQ43Wl1e_ONBQoRfJH5qhLUKKKugPxFKbnCvCeuZh1rrGW3Q2jlojiY049MqCT4EsAjf3nAcoqH3Xb2pxKJhg4PgL4Lbq4zxtQYwWiF7l0YVHtg9lna_8_mU5MIomoDa9xKm1pizJ4Q2p0IJ29B-PpCJihS4xCKIwtxt0q2KP5CNrDgj7IhEAxcERMQyoiZLn7n6sluSIaBdAn6E1cR_hOa3wOwdklisMPP5FTvUjsVWgYznZTBLSijglfvmnu7lq3MbMayn-ZYTMhALgJfVYoMHtleKAGfmOMUBTHElk-vLK6-evL49aMnSe2ycvAg9ph9B4bNQpdv6oDCyjMEgw3r1f3OE4uVxPiBhPgFRS7u4POCuCSDkFwP19zNT3KVKAhapD5QAdwS28cH3BfAkdfi0TIAMp39Kt-AUFGOPwxyxSDP-olUBEIt3K5vmWCvYFTHY_r3rdHqthdI-IzeQktoDJ4TJ0yAD8jDIx04tj0zm1MxhDlw42lN9HUFYLF84bkJfFy88nMCk5v-KIWvBjaxySgpBybte05Sph_LNnliijVI0RBJCL78x7R9KvTUf9pzwkfU9f7-_R6L3R2k68M3TtgUHclzkT2Ke3P3An15LzhIH7E5EBgrsBBIx1HoyBPdDi4KWEOhXgCufQN08rquMY-wy0oTFdX3yY4yQgLVDFwTZ8rABHxkQhNXCwuSKwkhJ5EiJVCAskW3V9iJor_b7QosoUAleLESSzw1oyJ6-LLK5cxDETCzscPBkdqnwhhIijHiuXz3T3J44oX06M_1GNL3ZRXz8BXw8ZPanQPcZEETzQgYFCIC0vdm9blv1OV-hT-Cv5X_6kZpKUB-pENQjxhFaQg93BwRUVh_kFNKkYIwJbjqmEQuVyqO1gxbwd6xvRS4brCN-R5yU25AetI8F7EpdGLsWHa0HJi53rk1I-A8qbRvlENdUCnalmNva2SeakfmIOoFkuRmo6hCTqLpBT-7uOqMlrEp_wEHwsH3YGx5kWGADe9KeHRRWlf5kP31CxPB5-woP1RcSkgiAnRIOYvETNJYeXcUWLlojuB7jii8tW1GPsj4rvaPtRTWhoWMgtDQcTM9BjBgpSUDbzoj38OZHeZ4ub1WNcAWfCKiC37q9e0IAmBT7odvMvOKcWjmICGUdyuFcplmKYrl-fx2p_HbxdxEV0lGmyTjUsECbpJm4lwF6_Y7lXFWaeOyYPGJtojf3a2wNrlizH3jBSgcXFaGUVlpEPeCBdzpKOS9A9250DL1HErRn_Ez2OGkdpNdX8GjwLupv6kTYytd3mrOd3u5LlcBOEid8HPyQLaKROkLUpXjUI8Cr-_4trUGKUzWpm7gjApUkiN-rEUH-qYU56TAHqbeCtwURhmiCf4Cu0J8j2D42SLAgSjvC4Bk1kbct29zQIBpTJq6AM72iScLKJZp8q_pkMSTMakWUke-7JcbrmMGR9pLh1JxBhZidJmnLl8Qo8mwnIgySi0i7uGY0cQtnLBsDrIef_Zi6aVAFziisMBuAGMgOp80Gu1eZ74r0i3jeIijCLH5dlpsKOmgP6U-RcFhZCfi0p5iRDRTlu9pkKyFQTIHMP_QSebWeM9zj7IOlyi548sICxtyvt0mnfJmmRAz0dYlIi3FmTyMB_CHXsEF_oP9Iuz3OubDpO88-3l9MYrXdl5F8z9vrjSGq8_mn7Sf7TnGm-fWcnfhVfhy3uyvPF7Z_uXh2uve3uL2yuLG373ce7S4-NPK89H8Tv_NZue5GS6tDv3Qi71k6edflt4Nfl3yhm5n-43XebjuvTHfeOGrhz8_9p8_3NuTm4KmcOiSoa02TFKkyexkHYNXSAnwonqk4REcLkM3tLUXi6trT1-uc2R8djM7UgCdW6ZmNClbpXN4-kop_cD6OmCoc1r71dtpNeabK1XhGc60coL7RsnFUQqAFAZx0KvXL148e_ETNCgpKZZRCTrmplmIU_k_p85gcw0dK8MiDuxUlBIrSTykT08jRz0P-uti1AqZ8wH0FOXShbYuuzK3pyS-otPnxV2OKeck5dhLI_wf200CKNdgtKszV8bbDqXzsJuQgIoQnHvT2_XtRF9jbf_QjG1rRfSHpp9CFJv27nQaSeSxv8gx72iA_oDG4qZk9KWdRV8UPuC8O3Y74iAIKYL7BGwC0kg3psSL1xuagxIQuSF9LYdsbgifRieDDUtBNv2MZnTI1TI683lKXwSOSyBTEvHVk0cvKf7ybRFCKuANha0LM0GPBXlIcf8wG9kWYaxIqB7BPr6YFU3cP0J2UShIUMGyRWZJAJ-zajoB0b7mXKfUa_fWV76f0G2_QJTOFwLWbE4vZC6qaMkdjDzQvFMZbJYeAJpfZXi1LS8vS8TTDRP7SSIcd9hY36lKbNMXDTyQRIAIxBWzZUs6jOsY8agw4DzTA8SC_8GHOy4SmRcmUnwAEVXpvdzzUpkErm7QN_uC3NczAZPoiC2WpP2UCNOptHwEbHNrH5ssjLEOjvd3vQBYxySPoBBYONEHiYe5gf9OyRHZWwgppHjIpUa70qvMxOTHGTIP8bLMyFrjwaJMaTfXpyKTSuY9Il8whq-b7eVzolifBbgU7ruzf1ReJicJAo9zW_7NLFs-YcrLfDXG5l2VX1LRLgr1w93zQFJCu1xnouIyaNvMyjW-r4Td1P43QewFcXHdTyfdfyH7KjkLr3JnV5TY4FqV2kafH_IShiznSHmFGMvahMGlDNMQK9vZ_RnUeAwsFTn9FKCxuLlrBdxcJaM_TJswhXRpz1wCloH2o80sjBuBqpQtEcz9FSkWVihrT-Zeiaa5J_6WEwW-xxGHH6trrAguY0-UjhuSCtuZCsbfy52EUKYQVf3AlKsgy9OUj-3EpihPq1XGsAFvTPexCPQRoZVjiyuQG5NOPdpOZzVS4JQCEMqLgypxNHvY2JsDLEXhRI6Ug-j7lyKBDFpeB3a6C5AriUFMui_Ile1L66tTkiFSJSSLs2MkC-TuTdet5dWD8hGsp7oh2tZXs63nQMRmfjh4fA9KVijdimId5rO5VddMhkHkzS2uPpus1LlTYZsk303lQumgA8_SAVwpyKQN5yYgLk6dvCVsm0cvvpqYf8bcnxDFt-2-nlL9x6f4ixjoPj5lvdylihqHXHffKnuhFMGGAXNBsJDN6FHKfF9hDJhlvMrVDNcraPg-FCHRQl5uJ_UzDiAUVDHU9YnRmG_Nt41es11ISQiUnqqiv25zwWh34J-dVw6wMEHyBJO5zqFjY7S46gtLz54sP36xuPLkLadAJ0iWF6mp5NO9UgloyUj1TUsWI8i8bW9WPi8MApfWD0Uh766SfLGnS5loYPAqCL84GmUGhK-DqgE9zhDswLMVdFR1uHMVo2k_r-iS81zEtUIkhpN1heReTxb05i50Vbbi8kSCU5lACubczIM5gm8IOkvdQB4T5z9dxy_nP8-oIrrZSe5uHvEthKFOdeeNRq_HVl1hUYHwznjOjm3NEbFyqc1rN6qUyMHCwKXqIRl0I2wpzfKXs8wyCyaVE4jvGnfQqEdcrjct5lAq3Y5LGfoRQIuxz5Hn-2MfVbgUk9v74D19kwN0wiai2CSH5wCdN2KqYn5_t8FBeg3GVDGTHOQkuH4KeVCboNj-Q5iepABZd5PvpSqzAeMBz09bOb66MdtnVhZRCStZfOVNypDxeTg6Ood236V2tCvjxTdLld-Vmv8khKLPwFKJI2uNrGSHZLFYLnEzY_2dzNHJy_zIX6ou1yT4L8VQ2I47U-elAVo-goPqjX1dDOkckz6I7EROrFdUiGs0XqPxWjZKE6Nk4J5eEchLXrsSv6iiCL61s8XkZ6kMf6L8keqsK8qSjRnFprGoQKABWp-Lz2iTAlHLcPKtCqr3XXwSYJR0v5ojT6mTQA--u98tLT5bfvJY-N0gDXCkrLnE0E4l1FG9sM7fRg6_CrJWHKjKTM3tROR4GXvDqpAuEUnf1bVV0fCcGooF9xmOYtWbxUYn0FkBdxOV1hRjVAJJOWYJLFToOPGuSSHvSIXCXLzip4Xs-InX60tajzJ8U-UoEpnJuMBRAhE_zoLFeVypukD_znvDLGTFTi-_eWn8-pZfv_gCSCzT3UTfLQpamQN5h0o45_ZLfQiX9qYoECF8VI8pwin6ckFNIt8kUs5SZi_PKq2m9Rvz1daFQznCfLFuIeN9Ni-EovAj4ACVOV8t1-_Y8DXiLN86VXf256yEqCBoAiZJ1MZq7FpxL6ws5OsOaO80VJkAnWqPa3aPvV5_JMrcqDhtBgSYq6rurPDTr2xSbZfMibAd5-xDocas2uEoAOjJNDVbdxXrTGrdaY7Puq6NbdflUOdr39n5iWFBFvRU9ZykDAWR595XIVSqV1Ypthn1yrKZVfW9abkt1k4XaXyyBZg9T5kTjrPCGBLBKiDUMQgFJJU6jfs4IgBSzCrt8lM68UPHh4P4KPApTUGPb1ckzLPidYWYSPjPiSLMnUTWX_7Ve-ovd5KCmcc9bJojwADZhO2UAryVvqox9bIAbWxn4NhbDNmOtxsL3clQ8X5Z0A5AqUCw-6Ve0Vyup_yqgMaFoWJBIU-iUBVwoVDOwYrj4l8m7XWBIHBwwe5MjgTzSCTF_enWCt2pC5UNNXTDAIm7vTrpIF0_2IwfIaOzQWVLWhcNzc5HW5SuA9BFM0NHa-odrdlo9IymYXzkU1Mm5mMfXT4GM3y0JSeTdh_nit1he4MvmJRL-yOvK6LxH3np2c0ffQswLV7gV-7iNXnM5W3EUMr-wK5D8_mETeqUUa5XvFikFZdnfCbzz6XD6c7WbjOblDFSabJBQL5TEMGk1MeRuWX7YkHxN5UB6619SWcF2z69YRxLdSv-S-ffCwLCJnaktfQW5qHBghaT9J1NoQNdRq7xLQDLA-h7PBzt9k1_U0r_j91e7kzPHLWxm4yxy5hTPS6QHKlLea73rpevpIHyY4Ax9YrXbx4thsdjE3iB7ZfEVoTpi7OXarXrK_Yjs7-PrWSQzk-x-EJDTwzxrTBlRZfSvb-_K2vD93cLoyAJ-ulQ2yh1_3945P2K69Vz8uGpfB-smTUebAbWWvSE0sBOHyq70W8dYihFPJONKGFd3zn0QAZbhx0rdEdMjRja_ICBhPYxtH34oZZpk94ycmN-mNG2q2XI4zDjCJfLDq0PWVdWEfDi7Q8fLuDEB00BJ5gJfqjNg6vpuIcYo9rIL4Gzmof3foN4yLZSfPG3TBe_cy0StB7x0CGYYQNuYAzeY7qYMbFS5zDcr8aTl48TfNAMMY1URYBaB8ONw4-X8Z7DD7eTuJWA9N3DsETBw-pSUUNq2ZF5-AmUz0R-Is7wm-fLnZ8PnAW8yKCofRgKpr6TaO3DKNvNYcyQ8BCrQF3AByDNbBxiIS8YbDpJoJmuy8JxiAWDKNDIveodfMj-LvCHzEB3aRxCMxY9tLbeOuxAJrLRPOzgHc-FW4Ab7RyGbeIQRzTU54cNFHSSZavXy2_B8nuisUyoXFIvwsqn4l3Y8-VXY7LaNJX-4aSsCNRS8YOZ8BPSN-gg30XOfr5H_cgPVblQeNQzKb17gr8ntZuTdWrifQRZCEEzXSsXWKYUp6QQl0Wx6zzxT9U2hap6kSX__sA_7yLrpPumS7-jotPbx9uOlYxX7WiNopWn2sZCu9Wg3MB18V4dv_bGb0TnP9NQu8zZv-zFuYVmo6VhyJmg8ObUuSaUamtBS-Z12-3kBZ_TlVdcxymCmrenI7BUWMwpplCEY0FU-Vrx9UJpILUQCFch9ZOdZrvZ6xV-rqAQSy2_vXV18qBZzrOWv65ZCKBjlzmhN5JKQqsQdCG3Qjwh3qsppOm4KOMz1bOYSbhw90_ZY5FEuJylq0aRGY6dQWz7W6Arv2uKp78axk-yoVQIdnOqlt-OoiDSI67HPtHQG42OKtTkH3_av2bldmXqZKIGuz4zv0UFolVF2FXT2juhU_qBtaz0hLPReUbg1r6-b1K7oophZE1K8bU8Onw5OZ3Vf-jhgLOfmM8YZhV173kjhiutC8nos8VrTGpPJn-DwPfv1z0_KTzp-1ofs6fhZEPsq5ZSSkm9NYbNO8Pd_L2xy-VUFLbqBBa_99QrvuAz62Xhih-2u0p0pESE-k07wUqFNNDt_PWKmEo-K15iAUW58--nUgXlxBTVDN2fLURcOsBKoVCHRXF_xUhZrcKt6Zcg5Y-osBqonSu9P5jnUC-q0lzxWinYa_KXP_Kfp-Iag1hpn2MGqdqySqekRWVahbp-O13mUP27Scf4ddQLMeyJToUIom5B7msn_F8lrw10",
+ "fs.s3n.block.size" : "67108864",
+ "pig.udf.context" : "eNqNUD1LxEAQnUQPcyqoF7le0MJmg40gByoowY8oilhd41yyJit7ybI70egv8Uf4AwQr8Q9oa23rf3BjiiBaODDDKx7vvXkPn9AxGhau8BpZSUKyPTTZEarO1PvzS__ibQLcEKZlgUmIMRV6H7qUaW6yQiaV2tqGemZvPHvn7TpWbK3QKUOFccaZEikTYyUb7fPdcKfIiVe03MJDfgvNOC64Q5hEnRoCfxjVoQKJeRqckRZ5OoigG0s05hjHnKD3m1ApRbD6l_-oFJJEzkKRJ6cl5jYONzbsYvv5iS4U1yS42ei93qn1zXsXnAi8hF9iKclm6jeONTto2YNKg_-zQMKR5P7T3MrSwcfjvyr0viucqWGlqi92oIBH",
+ "mapred.job.tracker.handler.count" : "64",
+ "keep.failed.task.files" : "false",
+ "mapred.output.compress" : "false",
+ "hadoop.security.group.mapping" : "org.apache.hadoop.security.ShellBasedUnixGroupsMapping",
+ "hadoop.home.dir" : "/usr/lib/hadoop-client-cdh3u2-t1",
+ "dfs.https.client.keystore.resource" : "ssl-client.xml",
+ "mapred.cache.files.filesizes" : "1795932",
+ "mapred.jobtracker.job.history.block.size" : "3145728",
+ "pig.tmpfilecompression.codec" : "lzo",
+ "pig.hbase.caster" : "HBaseBinaryConverter",
+ "mapred.skip.reduce.max.skip.groups" : "0",
+ "java.ext.dirs" : "/usr/java/jdk1.6.0_24/jre/lib/ext:/usr/java/packages/lib/ext",
+ "dfs.datanode.address" : "0.0.0.0:50010",
+ "dfs.datanode.max.xcievers" : "4096",
+ "dfs.datanode.https.address" : "0.0.0.0:50475",
+ "fs.s3.impl" : "org.apache.hadoop.fs.s3.S3FileSystem",
+ "pig.map.keytype" : "eNpb85aBtbSIgSnaac1n8R9sHCEPmBgYKgoYGBgYzQGOfgg3",
+ "ipc.server.read.threadpool.size" : "5",
+ "java.class.path" : "/etc/hadoop/conf:/usr/java/latest/lib/tools.jar:/etc/hbase/conf:/usr/lib/pig_0.11.0_68/bin/../conf:/usr/java/latest/lib/tools.jar:/home/billg/ws/git/ambrose/pig/target/ambrose-pig-0.1.1-SNAPSHOT-bin/ambrose-pig-0.1.1-SNAPSHOT/lib/jetty-6.1.25.jar:/home/billg/ws/git/ambrose/pig/target/ambrose-pig-0.1.1-SNAPSHOT-bin/ambrose-pig-0.1.1-SNAPSHOT/lib/servlet-api-2.5-20081211.jar:/home/billg/ws/git/ambrose/pig/target/ambrose-pig-0.1.1-SNAPSHOT-bin/ambrose-pig-0.1.1-SNAPSHOT/lib/jetty-util-6.1.25.jar:/home/billg/ws/git/ambrose/pig/target/ambrose-pig-0.1.1-SNAPSHOT-bin/ambrose-pig-0.1.1-SNAPSHOT/lib/pig-0.11.0.jar:/home/billg/ws/git/ambrose/pig/target/ambrose-pig-0.1.1-SNAPSHOT-bin/ambrose-pig-0.1.1-SNAPSHOT/lib/guava-12.0.1.jar:/home/billg/ws/git/ambrose/pig/target/ambrose-pig-0.1.1-SNAPSHOT-bin/ambrose-pig-0.1.1-SNAPSHOT/lib/slf4j-api-1.6.4.jar:/home/billg/ws/git/ambrose/pig/target/ambrose-pig-0.1.1-SNAPSHOT-bin/ambrose-pig-0.1.1-SNAPSHOT/lib/slf4j-simple-1.6.4.jar:/home/billg/ws/git/ambrose/pig/target/ambrose-pig-0.1.1-SNAPSHOT-bin/ambrose-pig-0.1.1-SNAPSHOT/lib/ambrose-pig-0.1.1-SNAPSHOT.jar:/home/billg/ws/git/ambrose/pig/target/ambrose-pig-0.1.1-SNAPSHOT-bin/ambrose-pig-0.1.1-SNAPSHOT/lib/ambrose-common-0.1.1-SNAPSHOT.jar:/Users/billg/ws/git/science/.pants.d/jars/com.twitter.ambrose-ambrose-pig-internal.jar:/home/billg/.ivy2/science/cache/com.twitter.corestorage/hraven/jars/hraven-0.2.3-SNAPSHOT.jar:/home/billg/ws/downloads/hbase/hbase/lib/zookeeper-3.3.2.jar:common/target/ambrose-common-0.1.1-SNAPSHOT.jar:pig/target/ambrose-pig-0.1.1-SNAPSHOT.jar:/etc/dal/conf:/usr/lib/pig_0.11.0_68/bin/../piggybank-0.11.0+68.jar:/usr/lib/pig_0.11.0_68/bin/../jython-standalone-2.5.2.jar:/usr/lib/pig_0.11.0_68/bin/../pig-0.11.0+68-withouthadoop.jar:/usr/lib/pig_0.11.0_68/bin/../pig-0.11.0+68-withouthadoop.jar::/usr/share/java/hadoop-lzo.jar:/usr/lib64/mecab/java/MeCab.jar:/etc/hbase/conf:/usr/local/hbase-0.90.t1/hbase.jar:/usr/local/hbase-0.90.t1/lib/zookeeper.jar:/usr/local/hbase-0.90.t1/lib/guava.jar:/usr/local/hbase-0.90.t1/lib/protobuf-java.jar:/usr/share/java/hadoop-lzo.jar:/usr/lib64/mecab/java/MeCab.jar:/etc/hbase/conf:/usr/local/hbase-0.90.t1/hbase.jar:/usr/local/hbase-0.90.t1/lib/zookeeper.jar:/usr/local/hbase-0.90.t1/lib/guava.jar:/usr/local/hbase-0.90.t1/lib/protobuf-java.jar:/usr/lib/hadoop-client-cdh3u2-t1/hadoop-core-0.20.2.cdh3u2-t1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/ant-contrib-1.0b3.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/aspectjrt-1.6.5.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/aspectjtools-1.6.5.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/commons-cli-1.2.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/commons-codec-1.4.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/commons-daemon-1.0.1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/commons-el-1.0.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/commons-httpclient-3.1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/commons-logging-1.0.4.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/commons-logging-api-1.0.4.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/commons-net-1.4.1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/core-3.1.1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/hadoop-capacity-scheduler-0.20.2.cdh3u2-t1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/hadoop-fairscheduler-0.20.2.cdh3u2-t1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/hsqldb-1.8.0.10.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/jackson-core-asl-1.5.2.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/jackson-mapper-asl-1.5.2.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/jasper-compiler-5.5.12.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/jasper-runtime-5.5.12.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/jets3t-0.6.1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/jetty-6.1.26.cloudera.1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/jetty-servlet-tester-6.1.26.cloudera.1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/jetty-util-6.1.26.cloudera.1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/jsch-0.1.42.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/junit-4.5.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/kfs-0.2.2.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/log4j-1.2.15.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/mockito-all-1.8.2.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/oro-2.0.8.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/servlet-api-2.5-20081211.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/servlet-api-2.5-6.1.14.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/slf4j-api-1.4.3.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/slf4j-log4j12-1.4.3.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/xmlenc-0.52.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/jsp-2.1/jsp-2.1.jar:/usr/lib/hadoop-client-cdh3u2-t1/lib/jsp-2.1/jsp-api-2.1.jar",
+ "ambrose.write.events.file" : "ambrose_events.json",
+ "mapred.jar" : "/user/billg/.staging/job_201301170859_1562015/job.jar",
+ "hadoop.tmp.dir" : "/tmp",
+ "mapred.line.input.format.linespermap" : "1",
+ "hadoop.kerberos.kinit.command" : "kinit",
+ "topology.script.number.args" : "1",
+ "dfs.datanode.du.reserved" : "107374182400",
+ "fs.default.name" : "hdfs://hadoop-dw-nn.smf1.twitter.com:8020",
+ "dfs.balance.bandwidthPerSec" : "41943040",
+ "pig.schematuple.local.dir" : "/tmp/1364953689203-0",
+ "os.version" : "2.6.39-t7.el5",
+ "mapred.local.dir.minspacestart" : "0",
+ "mapred.jobtracker.maxtasks.per.job" : "50000",
+ "mapred.user.jobconf.limit" : "5242880",
+ "mapred.reduce.max.attempts" : "4",
+ "pig.schematuple.classes" : "",
+ "mapred.job.tracker" : "hadoop-dw-jt.smf1.twitter.com:8021",
+ "dfs.namenode.decommission.interval" : "30",
+ "dfs.name.edits.dir" : "${dfs.name.dir}",
+ "pig.alias.location" : "M: L[1,4],L[-1,-1],JOINED[4,9],R[2,4],R[-1,-1],JOINED[4,9] C: R: ",
+ "java.awt.graphicsenv" : "sun.awt.X11GraphicsEnvironment",
+ "mapred.tasktracker.instrumentation" : "org.apache.hadoop.mapred.TaskTrackerMetricsInst",
+ "java.vm.vendor" : "Sun Microsystems Inc.",
+ "io.mapfile.bloom.error.rate" : "0.005",
+ "mapred.tasktracker.expiry.interval" : "600000",
+ "pig.home.dir" : "/usr/lib/pig_0.11.0_68/bin/..",
+ "io.sort.record.percent" : "0.05",
+ "dfs.safemode.threshold.pct" : "1.111f",
+ "mapred.job.tracker.persist.jobstatus.active" : "false",
+ "dfs.name.dir" : "/data/hadoop-dw-nn,/mnt/hadoop-dw-bn-backup,/mnt/hadoop-dw-snn-backup",
+ "mapreduce.job.acl-modify-job" : " ",
+ "fs.checkpoint.period" : "10800",
+ "mapred.fairscheduler.preemption.interval" : "30000",
+ "io.skip.checksum.errors" : "false",
+ "pig.optimistic.files.concatenation" : "false;",
+ "dfs.datanode.handler.count" : "8",
+ "pig.info.reducers.default.parallel" : "-1",
+ "dfs.namenode.decommission.nodes.per.interval" : "5",
+ "mapred.temp.dir" : "${hadoop.tmp.dir}/mapred/temp",
+ "file.encoding" : "UTF-8",
+ "hadoop.native.lib" : "true",
+ "mapreduce.job.counters.limit" : "120",
+ "dfs.datanode.ipc.address" : "0.0.0.0:50020",
+ "mapred.working.dir" : "hdfs://hadoop-dw-nn.smf1.twitter.com/user/billg",
+ "mapred.job.reduce.input.buffer.percent" : "0.0",
+ "pig.reduce.output.dirs" : "hdfs://hadoop-dw-nn.smf1.twitter.com/tmp/temp-977624468/tmp-1644509149",
+ "stop.on.failure" : "true"
+ },
+ "counterGroupMap" : {
+ "org.apache.hadoop.mapred.JobInProgress$Counter" : {
+ "groupName" : "org.apache.hadoop.mapred.JobInProgress$Counter",
+ "groupDisplayName" : "Job Counters ",
+ "counterInfoMap" : {
+ "SLOTS_MILLIS_MAPS" : {
+ "name" : "SLOTS_MILLIS_MAPS",
+ "displayName" : "SLOTS_MILLIS_MAPS",
+ "value" : 7058
+ },
+ "TOTAL_LAUNCHED_REDUCES" : {
+ "name" : "TOTAL_LAUNCHED_REDUCES",
+ "displayName" : "Launched reduce tasks",
+ "value" : 1
+ },
+ "FALLOW_SLOTS_MILLIS_REDUCES" : {
+ "name" : "FALLOW_SLOTS_MILLIS_REDUCES",
+ "displayName" : "Total time spent by all reduces waiting after reserving slots (ms)",
+ "value" : 0
+ },
+ "RACK_LOCAL_MAPS" : {
+ "name" : "RACK_LOCAL_MAPS",
+ "displayName" : "Rack-local map tasks",
+ "value" : 2
+ },
+ "FALLOW_SLOTS_MILLIS_MAPS" : {
+ "name" : "FALLOW_SLOTS_MILLIS_MAPS",
+ "displayName" : "Total time spent by all maps waiting after reserving slots (ms)",
+ "value" : 0
+ },
+ "TOTAL_LAUNCHED_MAPS" : {
+ "name" : "TOTAL_LAUNCHED_MAPS",
+ "displayName" : "Launched map tasks",
+ "value" : 2
+ },
+ "SLOTS_MILLIS_REDUCES" : {
+ "name" : "SLOTS_MILLIS_REDUCES",
+ "displayName" : "SLOTS_MILLIS_REDUCES",
+ "value" : 13025
+ }
+ }
+ },
+ "org.apache.pig.PigCounters" : {
+ "groupName" : "org.apache.pig.PigCounters",
+ "groupDisplayName" : "org.apache.pig.PigCounters",
+ "counterInfoMap" : {
+ "SPILLABLE_MEMORY_MANAGER_SPILL_COUNT" : {
+ "name" : "SPILLABLE_MEMORY_MANAGER_SPILL_COUNT",
+ "displayName" : "SPILLABLE_MEMORY_MANAGER_SPILL_COUNT",
+ "value" : 0
+ },
+ "PROACTIVE_SPILL_COUNT_RECS" : {
+ "name" : "PROACTIVE_SPILL_COUNT_RECS",
+ "displayName" : "PROACTIVE_SPILL_COUNT_RECS",
+ "value" : 0
+ },
+ "PROACTIVE_SPILL_COUNT_BAGS" : {
+ "name" : "PROACTIVE_SPILL_COUNT_BAGS",
+ "displayName" : "PROACTIVE_SPILL_COUNT_BAGS",
+ "value" : 0
+ }
+ }
+ },
+ "MultiInputCounters" : {
+ "groupName" : "MultiInputCounters",
+ "groupDisplayName" : "MultiInputCounters",
+ "counterInfoMap" : {
+ "Input records from _1_data1.txt" : {
+ "name" : "Input records from _1_data1.txt",
+ "displayName" : "Input records from _1_data1.txt",
+ "value" : 70
+ },
+ "Input records from _0_data1.txt" : {
+ "name" : "Input records from _0_data1.txt",
+ "displayName" : "Input records from _0_data1.txt",
+ "value" : 70
+ }
+ }
+ },
+ "FileSystemCounters" : {
+ "groupName" : "FileSystemCounters",
+ "groupDisplayName" : "FileSystemCounters",
+ "counterInfoMap" : {
+ "FILE_BYTES_READ" : {
+ "name" : "FILE_BYTES_READ",
+ "displayName" : "FILE_BYTES_READ",
+ "value" : 305
+ },
+ "HDFS_BYTES_READ" : {
+ "name" : "HDFS_BYTES_READ",
+ "displayName" : "HDFS_BYTES_READ",
+ "value" : 3786
+ },
+ "FILE_BYTES_WRITTEN" : {
+ "name" : "FILE_BYTES_WRITTEN",
+ "displayName" : "FILE_BYTES_WRITTEN",
+ "value" : 348555
+ },
+ "HDFS_BYTES_WRITTEN" : {
+ "name" : "HDFS_BYTES_WRITTEN",
+ "displayName" : "HDFS_BYTES_WRITTEN",
+ "value" : 91400
+ }
+ }
+ },
+ "MultiStoreCounters" : {
+ "groupName" : "MultiStoreCounters",
+ "groupDisplayName" : "MultiStoreCounters",
+ "counterInfoMap" : {
+ }
+ },
+ "org.apache.hadoop.mapred.Task$Counter" : {
+ "groupName" : "org.apache.hadoop.mapred.Task$Counter",
+ "groupDisplayName" : "Map-Reduce Framework",
+ "counterInfoMap" : {
+ "REDUCE_INPUT_GROUPS" : {
+ "name" : "REDUCE_INPUT_GROUPS",
+ "displayName" : "Reduce input groups",
+ "value" : 4
+ },
+ "COMBINE_OUTPUT_RECORDS" : {
+ "name" : "COMBINE_OUTPUT_RECORDS",
+ "displayName" : "Combine output records",
+ "value" : 0
+ },
+ "MAP_INPUT_RECORDS" : {
+ "name" : "MAP_INPUT_RECORDS",
+ "displayName" : "Map input records",
+ "value" : 140
+ },
+ "REDUCE_SHUFFLE_BYTES" : {
+ "name" : "REDUCE_SHUFFLE_BYTES",
+ "displayName" : "Reduce shuffle bytes",
+ "value" : 213
+ },
+ "REDUCE_OUTPUT_RECORDS" : {
+ "name" : "REDUCE_OUTPUT_RECORDS",
+ "displayName" : "Reduce output records",
+ "value" : 1500
+ },
+ "SPILLED_RECORDS" : {
+ "name" : "SPILLED_RECORDS",
+ "displayName" : "Spilled Records",
+ "value" : 280
+ },
+ "MAP_OUTPUT_BYTES" : {
+ "name" : "MAP_OUTPUT_BYTES",
+ "displayName" : "Map output bytes",
+ "value" : 4180
+ },
+ "SPLIT_RAW_BYTES" : {
+ "name" : "SPLIT_RAW_BYTES",
+ "displayName" : "SPLIT_RAW_BYTES",
+ "value" : 786
+ },
+ "MAP_OUTPUT_RECORDS" : {
+ "name" : "MAP_OUTPUT_RECORDS",
+ "displayName" : "Map output records",
+ "value" : 140
+ },
+ "COMBINE_INPUT_RECORDS" : {
+ "name" : "COMBINE_INPUT_RECORDS",
+ "displayName" : "Combine input records",
+ "value" : 0
+ },
+ "REDUCE_INPUT_RECORDS" : {
+ "name" : "REDUCE_INPUT_RECORDS",
+ "displayName" : "Reduce input records",
+ "value" : 140
+ }
+ }
+ }
+ },
+ "runtime" : "pig"
+ },
+ "successorNames" : [ "scope-48" ]
}, {
"name" : "scope-48",
- "aliases" : [ "ORDERED" ],
- "features" : [ "SAMPLER" ],
- "jobId" : "job_201204041958_199293",
- "successorNames" : [ "scope-63" ],
- "runtime" : "pig",
- "y" : -60.0,
- "x" : 120.0,
- "dagLevel" : 1
+ "job" : {
+ "id" : "job_201301170859_1562019",
+ "aliases" : [ "ORDERED" ],
+ "features" : [ "SAMPLER" ],
+ "mapReduceJobState" : {
+ "successful" : true,
+ "jobId" : "job_201301170859_1562019",
+ "complete" : true,
+ "jobName" : "PigLatin:exec.pig",
+ "trackingURL" : "http://hadoop-dw-jt.smf1.twitter.com:50030/jobdetails.jsp?jobid=job_201301170859_1562019",
+ "mapProgress" : 1.0,
+ "reduceProgress" : 1.0,
+ "totalMappers" : 1,
+ "totalReducers" : 1
+ },
+ "metrics" : {
+ "proactiveSpillCountObjects" : 0,
+ "avgMapTime" : 3427,
+ "hdfsBytesWritten" : 55,
+ "maxReduceTime" : 17416,
+ "minReduceTime" : 17416,
+ "SMMSpillCount" : 0,
+ "proactiveSpillCountRecs" : 0,
+ "numberReduces" : 1,
+ "avgReduceTime" : 17416,
+ "mapInputRecords" : 100,
+ "numberMaps" : 1,
+ "bytesWritten" : 0,
+ "maxMapTime" : 3427,
+ "recordWritten" : 0,
+ "mapOutputRecords" : 100,
+ "reduceOutputRecords" : 1,
+ "reduceInputRecords" : 100,
+ "minMapTime" : 3427
+ },
+ "configuration" : {
+ },
+ "counterGroupMap" : {
+ "org.apache.hadoop.mapred.JobInProgress$Counter" : {
+ "groupName" : "org.apache.hadoop.mapred.JobInProgress$Counter",
+ "groupDisplayName" : "Job Counters ",
+ "counterInfoMap" : {
+ "SLOTS_MILLIS_MAPS" : {
+ "name" : "SLOTS_MILLIS_MAPS",
+ "displayName" : "SLOTS_MILLIS_MAPS",
+ "value" : 4926
+ },
+ "TOTAL_LAUNCHED_REDUCES" : {
+ "name" : "TOTAL_LAUNCHED_REDUCES",
+ "displayName" : "Launched reduce tasks",
+ "value" : 1
+ },
+ "FALLOW_SLOTS_MILLIS_REDUCES" : {
+ "name" : "FALLOW_SLOTS_MILLIS_REDUCES",
+ "displayName" : "Total time spent by all reduces waiting after reserving slots (ms)",
+ "value" : 0
+ },
+ "RACK_LOCAL_MAPS" : {
+ "name" : "RACK_LOCAL_MAPS",
+ "displayName" : "Rack-local map tasks",
+ "value" : 1
+ },
+ "FALLOW_SLOTS_MILLIS_MAPS" : {
+ "name" : "FALLOW_SLOTS_MILLIS_MAPS",
+ "displayName" : "Total time spent by all maps waiting after reserving slots (ms)",
+ "value" : 0
+ },
+ "TOTAL_LAUNCHED_MAPS" : {
+ "name" : "TOTAL_LAUNCHED_MAPS",
+ "displayName" : "Launched map tasks",
+ "value" : 1
+ },
+ "SLOTS_MILLIS_REDUCES" : {
+ "name" : "SLOTS_MILLIS_REDUCES",
+ "displayName" : "SLOTS_MILLIS_REDUCES",
+ "value" : 13764
+ }
+ }
+ },
+ "org.apache.pig.PigCounters" : {
+ "groupName" : "org.apache.pig.PigCounters",
+ "groupDisplayName" : "org.apache.pig.PigCounters",
+ "counterInfoMap" : {
+ "SPILLABLE_MEMORY_MANAGER_SPILL_COUNT" : {
+ "name" : "SPILLABLE_MEMORY_MANAGER_SPILL_COUNT",
+ "displayName" : "SPILLABLE_MEMORY_MANAGER_SPILL_COUNT",
+ "value" : 0
+ },
+ "PROACTIVE_SPILL_COUNT_RECS" : {
+ "name" : "PROACTIVE_SPILL_COUNT_RECS",
+ "displayName" : "PROACTIVE_SPILL_COUNT_RECS",
+ "value" : 0
+ },
+ "PROACTIVE_SPILL_COUNT_BAGS" : {
+ "name" : "PROACTIVE_SPILL_COUNT_BAGS",
+ "displayName" : "PROACTIVE_SPILL_COUNT_BAGS",
+ "value" : 0
+ }
+ }
+ },
+ "MultiInputCounters" : {
+ "groupName" : "MultiInputCounters",
+ "groupDisplayName" : "MultiInputCounters",
+ "counterInfoMap" : {
+ }
+ },
+ "FileSystemCounters" : {
+ "groupName" : "FileSystemCounters",
+ "groupDisplayName" : "FileSystemCounters",
+ "counterInfoMap" : {
+ "FILE_BYTES_READ" : {
+ "name" : "FILE_BYTES_READ",
+ "displayName" : "FILE_BYTES_READ",
+ "value" : 105
+ },
+ "HDFS_BYTES_READ" : {
+ "name" : "HDFS_BYTES_READ",
+ "displayName" : "HDFS_BYTES_READ",
+ "value" : 91802
+ },
+ "FILE_BYTES_WRITTEN" : {
+ "name" : "FILE_BYTES_WRITTEN",
+ "displayName" : "FILE_BYTES_WRITTEN",
+ "value" : 237074
+ },
+ "HDFS_BYTES_WRITTEN" : {
+ "name" : "HDFS_BYTES_WRITTEN",
+ "displayName" : "HDFS_BYTES_WRITTEN",
+ "value" : 55
+ }
+ }
+ },
+ "MultiStoreCounters" : {
+ "groupName" : "MultiStoreCounters",
+ "groupDisplayName" : "MultiStoreCounters",
+ "counterInfoMap" : {
+ }
+ },
+ "org.apache.hadoop.mapred.Task$Counter" : {
+ "groupName" : "org.apache.hadoop.mapred.Task$Counter",
+ "groupDisplayName" : "Map-Reduce Framework",
+ "counterInfoMap" : {
+ "REDUCE_INPUT_GROUPS" : {
+ "name" : "REDUCE_INPUT_GROUPS",
+ "displayName" : "Reduce input groups",
+ "value" : 1
+ },
+ "COMBINE_OUTPUT_RECORDS" : {
+ "name" : "COMBINE_OUTPUT_RECORDS",
+ "displayName" : "Combine output records",
+ "value" : 0
+ },
+ "MAP_INPUT_RECORDS" : {
+ "name" : "MAP_INPUT_RECORDS",
+ "displayName" : "Map input records",
+ "value" : 100
+ },
+ "REDUCE_SHUFFLE_BYTES" : {
+ "name" : "REDUCE_SHUFFLE_BYTES",
+ "displayName" : "Reduce shuffle bytes",
+ "value" : 0
+ },
+ "REDUCE_OUTPUT_RECORDS" : {
+ "name" : "REDUCE_OUTPUT_RECORDS",
+ "displayName" : "Reduce output records",
+ "value" : 1
+ },
+ "SPILLED_RECORDS" : {
+ "name" : "SPILLED_RECORDS",
+ "displayName" : "Spilled Records",
+ "value" : 200
+ },
+ "MAP_OUTPUT_BYTES" : {
+ "name" : "MAP_OUTPUT_BYTES",
+ "displayName" : "Map output bytes",
+ "value" : 2662
+ },
+ "SPLIT_RAW_BYTES" : {
+ "name" : "SPLIT_RAW_BYTES",
+ "displayName" : "SPLIT_RAW_BYTES",
+ "value" : 402
+ },
+ "MAP_OUTPUT_RECORDS" : {
+ "name" : "MAP_OUTPUT_RECORDS",
+ "displayName" : "Map output records",
+ "value" : 100
+ },
+ "COMBINE_INPUT_RECORDS" : {
+ "name" : "COMBINE_INPUT_RECORDS",
+ "displayName" : "Combine input records",
+ "value" : 0
+ },
+ "REDUCE_INPUT_RECORDS" : {
+ "name" : "REDUCE_INPUT_RECORDS",
+ "displayName" : "Reduce input records",
+ "value" : 100
+ }
+ }
+ }
+ },
+ "runtime" : "pig"
+ },
+ "successorNames" : [ "scope-63" ]
}, {
"name" : "scope-63",
- "aliases" : [ "ORDERED" ],
- "features" : [ "ORDER_BY" ],
- "jobId" : "job_201204041958_199296",
- "successorNames" : [ ],
- "runtime" : "pig",
- "y" : -60.0,
- "x" : 240.0,
- "dagLevel" : 2
+ "job" : {
+ "id" : "job_201301170859_1562039",
+ "aliases" : [ "ORDERED" ],
+ "features" : [ "ORDER_BY" ],
+ "mapReduceJobState" : {
+ "successful" : true,
+ "jobId" : "job_201301170859_1562039",
+ "complete" : true,
+ "jobName" : "PigLatin:exec.pig",
+ "trackingURL" : "http://hadoop-dw-jt.smf1.twitter.com:50030/jobdetails.jsp?jobid=job_201301170859_1562039",
+ "mapProgress" : 1.0,
+ "reduceProgress" : 1.0,
+ "totalMappers" : 1,
+ "totalReducers" : 1
+ },
+ "metrics" : {
+ "proactiveSpillCountObjects" : 0,
+ "avgMapTime" : 15368,
+ "hdfsBytesWritten" : 68600,
+ "maxReduceTime" : 48071,
+ "minReduceTime" : 48071,
+ "SMMSpillCount" : 0,
+ "proactiveSpillCountRecs" : 0,
+ "numberReduces" : 1,
+ "avgReduceTime" : 48071,
+ "mapInputRecords" : 1500,
+ "numberMaps" : 1,
+ "bytesWritten" : 68600,
+ "maxMapTime" : 15368,
+ "recordWritten" : 1500,
+ "mapOutputRecords" : 1500,
+ "reduceOutputRecords" : 1500,
+ "reduceInputRecords" : 1500,
+ "minMapTime" : 15368
+ },
+ "configuration" : {
+ },
+ "counterGroupMap" : {
+ "org.apache.hadoop.mapred.JobInProgress$Counter" : {
+ "groupName" : "org.apache.hadoop.mapred.JobInProgress$Counter",
+ "groupDisplayName" : "Job Counters ",
+ "counterInfoMap" : {
+ "SLOTS_MILLIS_MAPS" : {
+ "name" : "SLOTS_MILLIS_MAPS",
+ "displayName" : "SLOTS_MILLIS_MAPS",
+ "value" : 8372
+ },
+ "TOTAL_LAUNCHED_REDUCES" : {
+ "name" : "TOTAL_LAUNCHED_REDUCES",
+ "displayName" : "Launched reduce tasks",
+ "value" : 1
+ },
+ "FALLOW_SLOTS_MILLIS_REDUCES" : {
+ "name" : "FALLOW_SLOTS_MILLIS_REDUCES",
+ "displayName" : "Total time spent by all reduces waiting after reserving slots (ms)",
+ "value" : 0
+ },
+ "RACK_LOCAL_MAPS" : {
+ "name" : "RACK_LOCAL_MAPS",
+ "displayName" : "Rack-local map tasks",
+ "value" : 1
+ },
+ "FALLOW_SLOTS_MILLIS_MAPS" : {
+ "name" : "FALLOW_SLOTS_MILLIS_MAPS",
+ "displayName" : "Total time spent by all maps waiting after reserving slots (ms)",
+ "value" : 0
+ },
+ "TOTAL_LAUNCHED_MAPS" : {
+ "name" : "TOTAL_LAUNCHED_MAPS",
+ "displayName" : "Launched map tasks",
+ "value" : 1
+ },
+ "SLOTS_MILLIS_REDUCES" : {
+ "name" : "SLOTS_MILLIS_REDUCES",
+ "displayName" : "SLOTS_MILLIS_REDUCES",
+ "value" : 41169
+ }
+ }
+ },
+ "org.apache.pig.PigCounters" : {
+ "groupName" : "org.apache.pig.PigCounters",
+ "groupDisplayName" : "org.apache.pig.PigCounters",
+ "counterInfoMap" : {
+ "SPILLABLE_MEMORY_MANAGER_SPILL_COUNT" : {
+ "name" : "SPILLABLE_MEMORY_MANAGER_SPILL_COUNT",
+ "displayName" : "SPILLABLE_MEMORY_MANAGER_SPILL_COUNT",
+ "value" : 0
+ },
+ "PROACTIVE_SPILL_COUNT_RECS" : {
+ "name" : "PROACTIVE_SPILL_COUNT_RECS",
+ "displayName" : "PROACTIVE_SPILL_COUNT_RECS",
+ "value" : 0
+ },
+ "PROACTIVE_SPILL_COUNT_BAGS" : {
+ "name" : "PROACTIVE_SPILL_COUNT_BAGS",
+ "displayName" : "PROACTIVE_SPILL_COUNT_BAGS",
+ "value" : 0
+ }
+ }
+ },
+ "MultiInputCounters" : {
+ "groupName" : "MultiInputCounters",
+ "groupDisplayName" : "MultiInputCounters",
+ "counterInfoMap" : {
+ }
+ },
+ "FileSystemCounters" : {
+ "groupName" : "FileSystemCounters",
+ "groupDisplayName" : "FileSystemCounters",
+ "counterInfoMap" : {
+ "FILE_BYTES_READ" : {
+ "name" : "FILE_BYTES_READ",
+ "displayName" : "FILE_BYTES_READ",
+ "value" : 896
+ },
+ "HDFS_BYTES_READ" : {
+ "name" : "HDFS_BYTES_READ",
+ "displayName" : "HDFS_BYTES_READ",
+ "value" : 91802
+ },
+ "FILE_BYTES_WRITTEN" : {
+ "name" : "FILE_BYTES_WRITTEN",
+ "displayName" : "FILE_BYTES_WRITTEN",
+ "value" : 233527
+ },
+ "HDFS_BYTES_WRITTEN" : {
+ "name" : "HDFS_BYTES_WRITTEN",
+ "displayName" : "HDFS_BYTES_WRITTEN",
+ "value" : 68600
+ }
+ }
+ },
+ "MultiStoreCounters" : {
+ "groupName" : "MultiStoreCounters",
+ "groupDisplayName" : "MultiStoreCounters",
+ "counterInfoMap" : {
+ }
+ },
+ "org.apache.hadoop.mapred.Task$Counter" : {
+ "groupName" : "org.apache.hadoop.mapred.Task$Counter",
+ "groupDisplayName" : "Map-Reduce Framework",
+ "counterInfoMap" : {
+ "REDUCE_INPUT_GROUPS" : {
+ "name" : "REDUCE_INPUT_GROUPS",
+ "displayName" : "Reduce input groups",
+ "value" : 4
+ },
+ "COMBINE_OUTPUT_RECORDS" : {
+ "name" : "COMBINE_OUTPUT_RECORDS",
+ "displayName" : "Combine output records",
+ "value" : 0
+ },
+ "MAP_INPUT_RECORDS" : {
+ "name" : "MAP_INPUT_RECORDS",
+ "displayName" : "Map input records",
+ "value" : 1500
+ },
+ "REDUCE_SHUFFLE_BYTES" : {
+ "name" : "REDUCE_SHUFFLE_BYTES",
+ "displayName" : "Reduce shuffle bytes",
+ "value" : 0
+ },
+ "REDUCE_OUTPUT_RECORDS" : {
+ "name" : "REDUCE_OUTPUT_RECORDS",
+ "displayName" : "Reduce output records",
+ "value" : 1500
+ },
+ "SPILLED_RECORDS" : {
+ "name" : "SPILLED_RECORDS",
+ "displayName" : "Spilled Records",
+ "value" : 3000
+ },
+ "MAP_OUTPUT_BYTES" : {
+ "name" : "MAP_OUTPUT_BYTES",
+ "displayName" : "Map output bytes",
+ "value" : 89900
+ },
+ "SPLIT_RAW_BYTES" : {
+ "name" : "SPLIT_RAW_BYTES",
+ "displayName" : "SPLIT_RAW_BYTES",
+ "value" : 402
+ },
+ "MAP_OUTPUT_RECORDS" : {
+ "name" : "MAP_OUTPUT_RECORDS",
+ "displayName" : "Map output records",
+ "value" : 1500
+ },
+ "COMBINE_INPUT_RECORDS" : {
+ "name" : "COMBINE_INPUT_RECORDS",
+ "displayName" : "Combine input records",
+ "value" : 0
+ },
+ "REDUCE_INPUT_RECORDS" : {
+ "name" : "REDUCE_INPUT_RECORDS",
+ "displayName" : "Reduce input records",
+ "value" : 1500
+ }
+ }
+ }
+ },
+ "runtime" : "pig"
+ },
+ "successorNames" : [ ]
} ]
6,856 common/src/main/resources/web/data/small-events.json
View
6,042 additions, 814 deletions not shown
34 common/src/main/resources/web/js/ambrose/core.js
View
@@ -24,16 +24,16 @@ define(['jquery'], function($) {
};
// pattern for separator char + alpha
- var rSepAlpha = /[-_ ]+([a-z])/ig;
+ var separatorAlphaPattern = /[-_ ]+([a-z])/ig;
// util function used by camelCase
- var fToUpper = function(all, match) {
+ function toUpper(all, match) {
return match.toUpperCase();
- };
+ }
// camel case string
String.prototype.camelCase = function() {
- return this.replace(rSepAlpha, fToUpper);
+ return this.replace(separatorAlphaPattern, toUpper);
};
// find max value within array
@@ -53,28 +53,6 @@ define(['jquery'], function($) {
return this.splice(i, 1);
};
- // core Ambrose object containing utility methods
- return {
- isNull: function(v) { return v == null; },
- notNull: function(v) { return v != null; },
-
- /**
- * @param array values to join.
- * @return string containing array values delimited by ', '.
- */
- commaDelimit: function(array) {
- if (array == null) return '';
- return array.join(', ');
- },
-
- /**
- * @param totalTasks total number of tasks.
- * @param taskProgress number in [0, 1].
- * @return formatted message.
- */
- taskProgressMessage: function(totalTasks, taskProgress) {
- if (totalTasks == null || taskProgress == null) return '';
- return totalTasks + ' (' + Math.round(Number(taskProgress) * 100, 0) + '%)';
- },
- };
+ // core Ambrose object
+ return {};
});
2  common/src/main/resources/web/js/ambrose/view/graph.js
View
@@ -62,7 +62,7 @@ define(['jquery', 'd3', '../core', './core'], function(
self.handleJobsUpdated([job], 350);
});
workflow.on('jobSelected jobMouseOver', function(event, job, prev) {
- self.handleJobsUpdated($.grep([prev, job], Ambrose.notNull));
+ self.handleJobsUpdated($.grep([prev, job], function(j) { j != null; }));
});
},
49 common/src/main/resources/web/js/ambrose/view/table.js
View
@@ -122,40 +122,57 @@ define(['jquery', 'd3', '../core', './core'], function($, d3, Ambrose, View) {
// update mutable row properties
var colors = this.params.colors;
if (duration) {
- // only update background color of rows whose jobs are not selected or mouseover
+ // rows updated due to event; transition background color gradually
tr.transition().duration(duration).filter(function(job) {
+ // don't update rows whose jobs are selected or mouseover
return !(job.mouseover || job.selected);
}).style('background-color', function(job) {
- if (job.status == 'RUNNING') return colors.running;
- if (job.status == 'COMPLETE') return colors.complete;
- if (job.status == 'FAILED') return colors.failed;
- return 'white';
+ var status = job.status || '';
+ return colors[status.toLowerCase()] || 'white';
});
+
} else {
// rows updated due to user interaction; rapidly update background color
tr.style('background-color', function(job) {
if (job.mouseover) return colors.mouseover;
if (job.selected) return colors.selected;
- if (job.status == 'RUNNING') return colors.running;
- if (job.status == 'COMPLETE') return colors.complete;
- if (job.status == 'FAILED') return colors.failed;
- return 'white';
+ var status = job.status || '';
+ return colors[status.toLowerCase()] || 'white';
});
}
+
+ function commaDelimit(array) {
+ if (array == null) return '';
+ return array.join(', ');
+ }
+
+ function taskProgressMessage(totalTasks, taskProgress) {
+ if (totalTasks == null || taskProgress == null) return '';
+ return totalTasks + ' (' + Math.round(Number(taskProgress) * 100, 0) + '%)';
+ }
+
// update all other params normally
tr.selectAll('a.job-url')
- .attr('href', function(job) { return job.trackingUrl || 'javascript:void(0);'; })
+ .attr('href', function(job) {
+ return job.mapReduceJobState.trackingURL || 'javascript:void(0);';
+ })
.text(function(job) { return job.id || 'pending'; });
tr.selectAll('td.job-status')
.text(function (job) { return job.status || ''; });
tr.selectAll('td.job-aliases')
- .text(function (job) { return Ambrose.commaDelimit(job.aliases); });
+ .text(function (job) { return commaDelimit(job.aliases); });
tr.selectAll('td.job-features')
- .text(function (job) { return Ambrose.commaDelimit(job.features); });
- tr.selectAll('td.job-mappers')
- .text(function (job) { return Ambrose.taskProgressMessage(job.totalMappers, job.mapProgress); });
- tr.selectAll('td.job-reducers')
- .text(function (job) { return Ambrose.taskProgressMessage(job.totalReducers, job.reduceProgress); });
+ .text(function (job) { return commaDelimit(job.features); });
+ tr.selectAll('td.job-mappers').text(function (job) {
+ return taskProgressMessage(
+ job.mapReduceJobState.totalMappers,
+ job.mapReduceJobState.mapProgress);
+ });
+ tr.selectAll('td.job-reducers').text(function (job) {
+ return taskProgressMessage(
+ job.mapReduceJobState.totalReducers,
+ job.mapReduceJobState.reduceProgress);
+ });
},
};
52 common/src/main/resources/web/js/ambrose/workflow.js
View
@@ -119,14 +119,18 @@ define(['jquery', 'uri', './core', './client', './graph'], function(
}
// reset job indices
- var jobs = self.jobs = data;
+ var jobs = self.jobs = [];
var jobsByName = self.jobsByName = {};
var jobsById = self.jobsById = {};
// initialize job indices
- $.each(jobs, function(i, job) {
+ $.each(data, function(i, node) {
+ // retrieve job from node
+ var job = node.job;
+ jobs.push(job);
+
// index job by name
- var name = job.name;
+ var name = job.name = node.name;
if (name in jobsByName) {
console.error("Multiple jobs found with name '" + name + "':", self);
return;
@@ -134,9 +138,8 @@ define(['jquery', 'uri', './core', './client', './graph'], function(
jobsByName[name] = job;
// index job by id (if defined)
- if (job.jobId) {
- var id = job.id = job.jobId;
- delete job.jobId;
+ if (job.id) {
+ var id = job.id;
if (id in jobsById) {
console.error("Multiple jobs found with id '" + id + "':", self);
return;
@@ -145,13 +148,20 @@ define(['jquery', 'uri', './core', './client', './graph'], function(
}
});
- // initialize parent links
- $.each(jobs, function(i, job) {
+ // initialize parent-child references
+ $.each(data, function(i, node) {
+ var job = node.job;
var name = job.name;
- $.each(job.successorNames, function(i, childName) {
+ if (!job.parentNames) job.parentNames = [];
+ // TODO(Andy Schlaikjer): Rename node.successorNames to childNames
+ $.each(job.childNames = node.successorNames, function(i, childName) {
var child = jobsByName[childName];
- var predecessorNames = child.predecessorNames || (child.predecessorNames = []);
- predecessorNames.push(name);
+ if (!child) {
+ console.error("No job with name '" + childName + "' exists", self);
+ return;
+ }
+ var parentNames = child.parentNames || (child.parentNames = []);
+ parentNames.push(name);
});
});
@@ -159,7 +169,7 @@ define(['jquery', 'uri', './core', './client', './graph'], function(
var graph = self.graph = Graph({
data: jobs,
getId: function(d) { return d.name; },
- getParentIds: function(d) { return d.predecessorNames; },
+ getParentIds: function(d) { return d.parentNames; },
});
graph.sort();
graph.addPseudoNodes();
@@ -267,8 +277,8 @@ define(['jquery', 'uri', './core', './client', './graph'], function(
// validate event data
var id = event.eventId;
var type = event.eventType;
- var job = event.eventData;
- if (!id || !type || !job) {
+ var data = event.eventData;
+ if (!id || !type || !data) {
console.error('Invalid event data:', self, event);
return;
}
@@ -281,18 +291,14 @@ define(['jquery', 'uri', './core', './client', './graph'], function(
// check for workflow event
if (type == 'WORKFLOW_PROGRESS') {
- self.setProgress(job.workflowProgress);
+ self.setProgress(data.workflowProgress);
return;
}
- // collect job data; JOB_FINISHED and JOB_FAILED events contain job.jobData
- if (job.jobId == null && job.jobData != null && job.jobData.jobId != null) {
- var jobData = job.jobData;
- delete job.jobData;
- $.extend(true, job, jobData);
- }
- job.id = job.jobId;
- delete job.jobId;
+ // collect job data
+ var node = data;
+ var job = node.job;
+ job.name = node.name;
// retrieve and update job with new data
job = self.updateJob(job);
87 pig/src/main/java/com/twitter/ambrose/pig/AmbrosePigProgressNotificationListener.java
View
@@ -17,6 +17,7 @@
import com.twitter.ambrose.model.Job;
import com.twitter.ambrose.model.WorkflowInfo;
+import com.twitter.ambrose.model.hadoop.MapReduceJobState;
import com.twitter.ambrose.service.DAGNode;
import com.twitter.ambrose.service.StatsWriteService;
import com.twitter.ambrose.service.WorkflowEvent;
@@ -64,7 +65,8 @@
private String workflowVersion;
private List<Job> jobs = new ArrayList<Job>();
- private Map<String, DAGNode> dagNodeNameMap = new TreeMap<String, DAGNode>();
+ private Map<String, DAGNode<PigJob>> dagNodeNameMap = new TreeMap<String, DAGNode<PigJob>>();
+ private Map<String, DAGNode<PigJob>> dagNodeJobIdMap = new TreeMap<String, DAGNode<PigJob>>();
private HashSet<String> completedJobIds = new HashSet<String>();
@@ -153,19 +155,16 @@ public void jobStartedNotification(String scriptId, String assignedJobId) {
for (JobStats jobStats : jobGraph) {
if (assignedJobId.equals(jobStats.getJobId())) {
log.info("jobStartedNotification - scope " + jobStats.getName() + " is jobId " + assignedJobId);
- DAGNode node = this.dagNodeNameMap.get(jobStats.getName());
+ DAGNode<PigJob> node = this.dagNodeNameMap.get(jobStats.getName());
if (node == null) {
- log.warn("jobStartedNotification - unrecorgnized operator name found ("
+ log.warn("jobStartedNotification - unrecognized operator name found ("
+ jobStats.getName() + ") for jobId " + assignedJobId);
} else {
node.getJob().setId(assignedJobId);
+ addMapReduceJobState(node.getJob());
+ dagNodeJobIdMap.put(node.getJob().getId(), node);
pushEvent(scriptId, WorkflowEvent.EVENT_TYPE.JOB_STARTED, node);
-
- Map<JobProgressField, String> progressMap = buildJobStatusMap(assignedJobId);
- if (progressMap != null) {
- pushEvent(scriptId, WorkflowEvent.EVENT_TYPE.JOB_PROGRESS, progressMap);
- }
}
}
}
@@ -178,9 +177,14 @@ public void jobStartedNotification(String scriptId, String assignedJobId) {
*/
@Override
public void jobFailedNotification(String scriptId, JobStats stats) {
- Job job = collectStats(scriptId, stats);
- jobs.add(job);
- pushEvent(scriptId, WorkflowEvent.EVENT_TYPE.JOB_FAILED, job);
+ DAGNode<PigJob> node = dagNodeJobIdMap.get(stats.getJobId());
+ if (node == null) {
+ log.warn("Unrecognized jobId reported for failed job: " + stats.getJobId());
+ return;
+ }
+
+ addCompletedJobStats(node.getJob(), stats);
+ pushEvent(scriptId, WorkflowEvent.EVENT_TYPE.JOB_FAILED, node);
}
/**
@@ -190,9 +194,14 @@ public void jobFailedNotification(String scriptId, JobStats stats) {
*/
@Override
public void jobFinishedNotification(String scriptId, JobStats stats) {
- Job job = collectStats(scriptId, stats);
- jobs.add(job);
- pushEvent(scriptId, WorkflowEvent.EVENT_TYPE.JOB_FINISHED, job);
+ DAGNode<PigJob> node = dagNodeJobIdMap.get(stats.getJobId());
+ if (node == null) {
+ log.warn("Unrecognized jobId reported for succeeded job: " + stats.getJobId());
+ return;
+ }
+
+ addCompletedJobStats(node.getJob(), stats);
+ pushEvent(scriptId, WorkflowEvent.EVENT_TYPE.JOB_FINISHED, node);
}
/**
@@ -232,17 +241,17 @@ public void progressUpdatedNotification(String scriptId, int progress) {
pushEvent(scriptId, WorkflowEvent.EVENT_TYPE.WORKFLOW_PROGRESS, eventData);
// then for each running job, we report the job progress
- for (DAGNode node : dagNodeNameMap.values()) {
+ for (DAGNode<PigJob> node : dagNodeNameMap.values()) {
// don't send progress events for unstarted jobs
if (node.getJob().getId() == null) { continue; }
- Map<JobProgressField, String> progressMap = buildJobStatusMap(node.getJob().getId());
+ addMapReduceJobState(node.getJob());
//only push job progress events for a completed job once
- if (progressMap != null && !completedJobIds.contains(node.getJob().getId())) {
- pushEvent(scriptId, WorkflowEvent.EVENT_TYPE.JOB_PROGRESS, progressMap);
+ if (node.getJob().getMapReduceJobState() != null && !completedJobIds.contains(node.getJob().getId())) {
+ pushEvent(scriptId, WorkflowEvent.EVENT_TYPE.JOB_PROGRESS, node);
- if ("true".equals(progressMap.get(JobProgressField.isComplete))) {
+ if (node.getJob().getMapReduceJobState().isComplete()) {
completedJobIds.add(node.getJob().getId());
}
}
@@ -259,14 +268,9 @@ public void jobsSubmittedNotification(String scriptId, int numJobsSubmitted) { }
public void outputCompletedNotification(String scriptId, OutputStats outputStats) { }
/**
- * Collects statistics from JobStats and builds a nested Map of values. Subsclass ond override
- * if you'd like to generate different stats.
- *
- * @param scriptId
- * @param stats
- * @return
+ * Collects statistics from JobStats and builds a nested Map of values.
*/
- protected Job collectStats(String scriptId, JobStats stats) {
+ private void addCompletedJobStats(PigJob job, JobStats stats) {
// put the job conf into a Properties object so we can serialize them
Properties jobConfProperties = new Properties();
@@ -283,7 +287,9 @@ protected Job collectStats(String scriptId, JobStats stats) {
}
}
- return new PigJob(stats, jobConfProperties);
+ job.setJobStats(stats);
+ job.setConfiguration(jobConfProperties);
+ jobs.add(job);
}
private void outputStatsData(WorkflowInfo workflowInfo) throws IOException {
@@ -301,36 +307,23 @@ private void pushEvent(String scriptId, WorkflowEvent.EVENT_TYPE eventType, Obje
}
@SuppressWarnings("deprecation")
- private Map<JobProgressField, String> buildJobStatusMap(String jobId) {
+ private void addMapReduceJobState(PigJob pigJob) {
JobClient jobClient = PigStats.get().getJobClient();
try {
- RunningJob rj = jobClient.getJob(jobId);
- if (rj == null) {
- log.warn("Couldn't find job status for jobId=" + jobId);
- return null;
+ RunningJob runningJob = jobClient.getJob(pigJob.getId());
+ if (runningJob == null) {
+ log.warn("Couldn't find job status for jobId=" + pigJob.getId());
+ return;
}
- JobID jobID = rj.getID();
+ JobID jobID = runningJob.getID();
TaskReport[] mapTaskReport = jobClient.getMapTaskReports(jobID);
TaskReport[] reduceTaskReport = jobClient.getReduceTaskReports(jobID);
- Map<JobProgressField, String> progressMap = new HashMap<JobProgressField, String>();
-
- progressMap.put(JobProgressField.jobId, jobId.toString());
- progressMap.put(JobProgressField.jobName, rj.getJobName());
- progressMap.put(JobProgressField.trackingUrl, rj.getTrackingURL());
- progressMap.put(JobProgressField.isComplete, Boolean.toString(rj.isComplete()));
- progressMap.put(JobProgressField.isSuccessful, Boolean.toString(rj.isSuccessful()));
- progressMap.put(JobProgressField.mapProgress, Float.toString(rj.mapProgress()));
- progressMap.put(JobProgressField.reduceProgress, Float.toString(rj.reduceProgress()));
- progressMap.put(JobProgressField.totalMappers, Integer.toString(mapTaskReport.length));
- progressMap.put(JobProgressField.totalReducers, Integer.toString(reduceTaskReport.length));
- return progressMap;
+ pigJob.setMapReduceJobState(new MapReduceJobState(runningJob, mapTaskReport, reduceTaskReport));
} catch (IOException e) {
log.error("Error getting job info.", e);
}
-
- return null;
}
private static String[] toArray(String string) {
58 pig/src/main/java/com/twitter/ambrose/pig/PigJob.java
View
@@ -17,6 +17,7 @@
import com.twitter.ambrose.model.Job;
import com.twitter.ambrose.model.hadoop.CounterGroup;
+import com.twitter.ambrose.model.hadoop.MapReduceJobState;
import org.apache.pig.tools.pigstats.InputStats;
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.OutputStats;
@@ -46,6 +47,7 @@
private String[] aliases;
private String[] features;
+ private MapReduceJobState mapReduceJobState;
private List<InputInfo> inputInfoList;
private List<OutputInfo> outputInfoList;
@@ -57,8 +59,39 @@ public PigJob(String[] aliases, String[] features) {
this.features = features;
}
- public PigJob(JobStats stats, Properties configuration) {
- super(RUNTIME, configuration);
+ @JsonCreator
+ public PigJob(@JsonProperty("id") String id,
+ @JsonProperty("aliases") String[] aliases,
+ @JsonProperty("features") String[] features,
+ @JsonProperty("mapReduceJobState") MapReduceJobState mapReduceJobState,
+ @JsonProperty("metrics") Map<String, Number> metrics,
+ @JsonProperty("configuration") Properties configuration,
+ @JsonProperty("counterGroupMap") Map<String, CounterGroup> counterGroupMap,
+ @JsonProperty("inputInfoList") List<InputInfo