Improve pysrc2cpg parse error handling. (#4629)

- Fix last error token position if error token itself is semicolon, newline or EOF. - Also log the parser exception message.
joernio · Jun 3, 2024 · 49f6b5f · 49f6b5f
1 parent befd101
commit 49f6b5f
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 4 deletions.
diff --git a/joern-cli/frontends/pysrc2cpg/pythonGrammar.jj b/joern-cli/frontends/pysrc2cpg/pythonGrammar.jj
@@ -34,11 +34,12 @@ public class PythonParser {
   }
 
   ErrorStatement recoverAndCreateErrorStmt(Token lastCorrectToken, Exception exception) {
-    Token prevToken = token;
+    Token lastErrorToken = null;
     try {
       getNextToken();
+      lastErrorToken = token;
       while (token.kind != SEMICOLON && token.kind != NEWLINE && token.kind != EOF) {
-        prevToken = token;
+        lastErrorToken = token;
         getNextToken();
       }
     } catch (Exception e) {
@@ -50,7 +51,7 @@ public class PythonParser {
     }
 
     Token errorStartToken = lastCorrectToken.next;
-    ErrorStatement errorStmt = new ErrorStatement(exception, attributes(errorStartToken, prevToken));
+    ErrorStatement errorStmt = new ErrorStatement(exception, attributes(errorStartToken, lastErrorToken));
     errors.add(errorStmt);
     return errorStmt;
   }

diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala
@@ -1293,7 +1293,10 @@ class PythonAstVisitor(
     val code   = nodeToCode.getCode(errorStatement)
     val line   = errorStatement.attributeProvider.lineno
     val column = errorStatement.attributeProvider.col_offset
-    logger.warn(s"Could not parse file $relFileName at line $line column $column. Invalid code: $code")
+    logger.warn(
+      s"Could not parse file $relFileName at line $line column $column. Invalid code: $code" +
+        s"\nParser exception message: ${errorStatement.exception.getMessage}"
+    )
     nodeBuilder.unknownNode(errorStatement.toString, errorStatement.getClass.getName, lineAndColOf(errorStatement))
   }