Skip to content
This repository has been archived by the owner on Jan 15, 2022. It is now read-only.

Commit

Permalink
Merge pull request #130 from vrushalic/add_rec_job_error
Browse files Browse the repository at this point in the history 
Add record type JOB_ERROR and SETUP_ATTEMPT_FINISHED
  • Loading branch information
@jrottinghuis
jrottinghuis committed Jan 9, 2015
2 parents ff97b69 + d68de93 commit 533094d
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 2 deletions.
6 changes: 4 additions & 2 deletions hraven-etl/src/main/java/com/twitter/hraven/etl/JobHistoryFileParserHadoop2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,10 +146,12 @@ public static enum Hadoop2RecordType {
JobStatusChanged("JOB_STATUS_CHANGED"),
JobQueueChange("JOB_QUEUE_CHANGED"),
JobSubmitted("JOB_SUBMITTED"),
JobUnsuccessfulCompletion("JOB_KILLED","JOB_FAILED"),
JobUnsuccessfulCompletion("JOB_KILLED","JOB_FAILED",
"JOB_ERROR"),
MapAttemptFinished("MAP_ATTEMPT_FINISHED"),
ReduceAttemptFinished("REDUCE_ATTEMPT_FINISHED"),
TaskAttemptFinished("CLEANUP_ATTEMPT_FINISHED"),
TaskAttemptFinished("CLEANUP_ATTEMPT_FINISHED",
"SETUP_ATTEMPT_FINISHED"),
TaskAttemptStarted("CLEANUP_ATTEMPT_STARTED",
"SETUP_ATTEMPT_STARTED",
"REDUCE_ATTEMPT_STARTED",
Expand Down
29 changes: 29 additions & 0 deletions hraven-etl/src/test/java/com/twitter/hraven/etl/TestJobHistoryFileParserHadoop2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,4 +311,33 @@ public void testVersion2_4() throws IOException {
// ensure that we got the hadoop2 version put
assertTrue(foundQueue);
}

@Test
public void testCreateJobHistoryFileParserJobError() throws IOException {
final String JOB_HISTORY_FILE_NAME = "src/test/resources/" +
"job_1300307629410_0001-1393307687476-user1-Example+job-1393307723835-0-0-ERROR-default-1393307693920.jhist";

File jobHistoryfile = new File(JOB_HISTORY_FILE_NAME);
byte[] contents = Files.toByteArray(jobHistoryfile);

JobHistoryFileParser historyFileParser =
JobHistoryFileParserFactory.createJobHistoryFileParser(contents, new Configuration(),
HistoryFileType.TWO);
assertNotNull(historyFileParser);

// confirm that we get back an object that can parse hadoop 2.0 files
assertTrue(historyFileParser instanceof JobHistoryFileParserHadoop2);

JobKey jobKey = new JobKey("cluster1", "user1", "Example job", 1, "job_1300307629410_0001");
// this would throw an exception when JOB_ERROR was not added as a record type
// check that processing goes ahead
historyFileParser.parse(contents, jobKey);
List<Put> jobPuts = historyFileParser.getJobPuts();
assertEquals(6, jobPuts.size());

JobKeyConverter jobKeyConv = new JobKeyConverter();
assertEquals("cluster1!user1!Example job!1!job_1300307629410_0001",
jobKeyConv.fromBytes(jobPuts.get(0).getRow()).toString());

}
}
19 changes: 19 additions & 0 deletions ..._0001-1393307687476-user1-Example+job-1393307723835-0-0-ERROR-default-1393307693920.jhist
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
Avro-Json
{"type":"record","name":"Event","namespace":"org.apache.hadoop.mapreduce.jobhistory","fields":[{"name":"type","type":{"type":"enum","name":"EventType","symbols":["JOB_SUBMITTED","JOB_INITED","JOB_FINISHED","JOB_PRIORITY_CHANGED","JOB_STATUS_CHANGED","JOB_FAILED","JOB_KILLED","JOB_ERROR","JOB_INFO_CHANGED","TASK_STARTED","TASK_FINISHED","TASK_FAILED","TASK_UPDATED","NORMALIZED_RESOURCE","MAP_ATTEMPT_STARTED","MAP_ATTEMPT_FINISHED","MAP_ATTEMPT_FAILED","MAP_ATTEMPT_KILLED","REDUCE_ATTEMPT_STARTED","REDUCE_ATTEMPT_FINISHED","REDUCE_ATTEMPT_FAILED","REDUCE_ATTEMPT_KILLED","SETUP_ATTEMPT_STARTED","SETUP_ATTEMPT_FINISHED","SETUP_ATTEMPT_FAILED","SETUP_ATTEMPT_KILLED","CLEANUP_ATTEMPT_STARTED","CLEANUP_ATTEMPT_FINISHED","CLEANUP_ATTEMPT_FAILED","CLEANUP_ATTEMPT_KILLED","AM_STARTED"]}},{"name":"event","type":[{"type":"record","name":"JobFinished","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"finishedReduces","type":"int"},{"name":"failedMaps","type":"int"},{"name":"failedReduces","type":"int"},{"name":"totalCounters","type":{"type":"record","name":"JhCounters","fields":[{"name":"name","type":"string"},{"name":"groups","type":{"type":"array","items":{"type":"record","name":"JhCounterGroup","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"counts","type":{"type":"array","items":{"type":"record","name":"JhCounter","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"value","type":"long"}]}}}]}}}]}},{"name":"mapCounters","type":"JhCounters"},{"name":"reduceCounters","type":"JhCounters"}]},{"type":"record","name":"JobInfoChange","fields":[{"name":"jobid","type":"string"},{"name":"submitTime","type":"long"},{"name":"launchTime","type":"long"}]},{"type":"record","name":"JobInited","fields":[{"name":"jobid","type":"string"},{"name":"launchTime","type":"long"},{"name":"totalMaps","type":"int"},{"name":"totalReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"uberized","type":"boolean"}]},{"type":"record","name":"AMStarted","fields":[{"name":"applicationAttemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"containerId","type":"string"},{"name":"nodeManagerHost","type":"string"},{"name":"nodeManagerPort","type":"int"},{"name":"nodeManagerHttpPort","type":"int"}]},{"type":"record","name":"JobPriorityChange","fields":[{"name":"jobid","type":"string"},{"name":"priority","type":"string"}]},{"type":"record","name":"JobStatusChanged","fields":[{"name":"jobid","type":"string"},{"name":"jobStatus","type":"string"}]},{"type":"record","name":"JobSubmitted","fields":[{"name":"jobid","type":"string"},{"name":"jobName","type":"string"},{"name":"userName","type":"string"},{"name":"submitTime","type":"long"},{"name":"jobConfPath","type":"string"},{"name":"acls","type":{"type":"map","values":"string"}},{"name":"jobQueueName","type":"string"},{"name":"workflowId","type":"string"},{"name":"workflowName","type":"string"},{"name":"workflowNodeName","type":"string"},{"name":"workflowAdjacencies","type":"string"},{"name":"workflowTags","type":"string"}]},{"type":"record","name":"JobUnsuccessfulCompletion","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"finishedReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"diagnostics","type":"string"}]},{"type":"record","name":"MapAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"mapFinishTime","type":"long"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"ReduceAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"shuffleFinishTime","type":"long"},{"name":"sortFinishTime","type":"long"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"finishTime","type":"long"},{"name":"rackname","type":"string"},{"name":"hostname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"}]},{"type":"record","name":"TaskAttemptStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"trackerName","type":"string"},{"name":"httpPort","type":"int"},{"name":"shufflePort","type":"int"},{"name":"containerId","type":"string"},{"name":"locality","type":"string"},{"name":"avataar","type":"string"}]},{"type":"record","name":"TaskAttemptUnsuccessfulCompletion","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"status","type":"string"},{"name":"error","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskFailed","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"error","type":"string"},{"name":"failedDueToAttempt","type":["null","string"]},{"name":"status","type":"string"},{"name":"counters","type":"JhCounters"}]},{"type":"record","name":"TaskFinished","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"status","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"successfulAttemptId","type":"string"}]},{"type":"record","name":"TaskStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"startTime","type":"long"},{"name":"splitLocations","type":"string"}]},{"type":"record","name":"TaskUpdated","fields":[{"name":"taskid","type":"string"},{"name":"finishTime","type":"long"}]}]}]}
{"type":"AM_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.AMStarted":{"applicationAttemptId":"appattempt_1393307629410_0001_000001","startTime":1393307691014,"containerId":"container_1393307629410_0001_01_000001","nodeManagerHost":"172.25.142.198","nodeManagerPort":57763,"nodeManagerHttpPort":8042}}}

{"type":"JOB_SUBMITTED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobSubmitted":{"jobid":"job_1300307629410_0001","jobName":"Example job","userName":"user1","submitTime":1393307687476,"jobConfPath":"hdfs://localhost:9000/tmp/hadoop-yarn/staging/user/.staging/job_1300307629410_0001/job.xml","acls":{},"jobQueueName":"default","workflowId":"","workflowName":"","workflowNodeName":"","workflowAdjacencies":"","workflowTags":""}}}

{"type":"JOB_INITED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInited":{"jobid":"job_1300307629410_0001","launchTime":1393307693920,"totalMaps":1,"totalReduces":0,"jobStatus":"INITED","uberized":false}}}

{"type":"JOB_INFO_CHANGED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInfoChange":{"jobid":"job_1300307629410_0001","submitTime":1393307687476,"launchTime":1393307693920}}}

{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1393307629410_0001_m_000000","taskType":"MAP","startTime":1393307693956,"splitLocations":""}}}

{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1393307629410_0001_m_000000","taskType":"MAP","attemptId":"attempt_1393307629410_0001_m_000000_0","startTime":1393307696163,"trackerName":"172.25.142.198","httpPort":8042,"shufflePort":13562,"containerId":"container_1393307629410_0001_01_000002","locality":"OFF_SWITCH","avataar":"VIRGIN"}}}

{"type":"MAP_ATTEMPT_FAILED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptUnsuccessfulCompletion":{"taskid":"task_1393307629410_0001_m_000000","taskType":"MAP","attemptId":"attempt_1393307629410_0001_m_000000_0","finishTime":1393307723803,"hostname":"172.25.142.198","port":57763,"rackname":"/default-rack","status":"FAILED","error":"AttemptID:attempt_1393307629410_0001_m_000000_0 Timed out after 5 secs","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":86662},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":48},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":1},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":0},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":48},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":13},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":85000192}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":0}]}]},"clockSplits":[747,747,748,747,747,748,747,747,748,747,747,748],"cpuUsages":[0,0,0,0,0,0,0,0,0,0,0,0],"vMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0],"physMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0]}}}

{"type":"TASK_FAILED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFailed":{"taskid":"task_1393307629410_0001_m_000000","taskType":"MAP","finishTime":1393307723803,"error":", AttemptID:attempt_1393307629410_0001_m_000000_0 Timed out after 5 secs","failedDueToAttempt":{"string":"attempt_1393307629410_0001_m_000000_0"},"status":"FAILED","counters":{"name":"COUNTERS","groups":[]}}}}

{"type":"JOB_ERROR","event":{"org.apache.hadoop.mapreduce.jobhistory.JobUnsuccessfulCompletion":{"jobid":"job_1300307629410_0001","finishTime":1420827068106,"finishedMaps":0,"finishedReduces":0,"jobStatus":"ERROR","diagnostics":"Task failed task_1393307629410_0001_m_000000_0 \nJob failed as tasks failed. failedMaps:1 failedReduces:0\nInvalid event T_ATTEMPT_KILLED on Task task_1393307629410_0001_m_000000_0 "}}}

0 comments on commit 533094d

Please sign in to comment.