Merge pull request #1787 from davidmarin/not-a-valid-jar

handle "Not a valid JAR" error message (fixes #1771)
Yelp · May 29, 2018 · a954048 · a954048
2 parents 250dad2 + d208a63
commit a954048
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 0 deletions.
diff --git a/mrjob/examples/mr_jar_step_example.py b/mrjob/examples/mr_jar_step_example.py
@@ -1,5 +1,6 @@
 # Copyright 2013 David Marin
 # Copyright 2016 Yelp
+# Copyright 2018 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/mrjob/logs/step.py b/mrjob/logs/step.py
@@ -71,6 +71,8 @@
 _JOB_PROGRESS_RE = re.compile(
     r'^\s*map\s+(?P<map>\d+)%\s+reduce\s+(?P<reduce>\d+)%\s*$')
 
+# if you specify a bad jar, this is all you get
+_NOT_A_VALID_JAR_RE = re.compile(r'^\s*Not a valid JAR:.*')
 
 # YARN prints this (sometimes followed by a Java exception) when tasks fail
 _TASK_ATTEMPT_FAILED_RE = re.compile(
@@ -316,6 +318,19 @@ def _parse_step_syslog_from_log4j_records(records, step_interpretation=None):
                 message=message,
             )
 
+        # invalid jar
+        m = _NOT_A_VALID_JAR_RE.match(message)
+        if m:
+            error = dict(
+                hadoop_error=dict(
+                    message=message,
+                    num_lines=record['num_lines'],
+                    start_line=record['start_line'],
+                ),
+            )
+            result.setdefault('errors', [])
+            result['errors'].append(error)
+
         # task failure
         m = _TASK_ATTEMPT_FAILED_RE.match(message)
         if m:

diff --git a/tests/logs/test_step.py b/tests/logs/test_step.py
@@ -98,6 +98,21 @@
     progress=dict(map=100, message=' map 100%  reduce 100%', reduce=100),
 )
 
+NOT_A_VALID_JAR_LOG_LINES = [
+    'Not a valid JAR: /home/hadoop/hadoop-examples.jar',
+]
+
+PARSED_NOT_A_VALID_JAR_LOG_LINES = dict(
+    errors=[
+        dict(
+            hadoop_error=dict(
+                message='Not a valid JAR: /home/hadoop/hadoop-examples.jar',
+                start_line=0,
+                num_lines=1,
+            ),
+        ),
+    ],
+)
 
 class ParseStepSyslogTestCase(TestCase):
 
@@ -114,6 +129,12 @@ def test_pre_yarn(self):
             _parse_step_syslog(PRE_YARN_STEP_LOG_LINES),
             PARSED_PRE_YARN_STEP_LOG_LINES)
 
+    def test_not_a_valid_jar(self):
+        self.assertEqual(
+            _parse_step_syslog(NOT_A_VALID_JAR_LOG_LINES),
+            PARSED_NOT_A_VALID_JAR_LOG_LINES,
+        )
+
 
 class InterpretHadoopJarCommandStderrTestCase(TestCase):