Merge pull request #797 from michaelpj/fix-recovery

Fix excess token consumption after `recoverInline`

runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java

@@ -63,6 +63,9 @@ public interface ANTLRErrorStrategy {
 	 * returns the {@link Token} instance which should be treated as the
 	 * successful result of the match.
 	 *
+   * <p>This method handles the consumption of any tokens - the caller should
+	 * <b>not</b> call {@link Parser#consume} after a successful recovery.</p>
+	 *
 	 * <p>Note that the calling code will not report an error if this method
 	 * returns successfully. The error strategy implementation is responsible
 	 * for calling {@link Parser#notifyErrorListeners} as appropriate.</p>

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -647,8 +647,9 @@ setState(<m.stateNumber>);
 <capture>
 if ( <if(invert)><m.varName> \<= 0 || <else>!<endif>(<expr>) ) {
 	<if(m.labels)><m.labels:{l | <labelref(l)> = (Token)}><endif>_errHandler.recoverInline(this);
+} else {
+	consume();
 }
-consume();
 >>
 
 Wildcard(w) ::= <<

tool/test/org/antlr/v4/test/rt/gen/Generator.java

@@ -585,6 +585,10 @@ private JUnitTestFile buildParserErrors() throws Exception {
 				"aab",
 				"",
 				"line 1:1 extraneous input 'a' expecting {'b', 'c'}\n");
+		file.addParserTest(input, "SingleTokenDeletionConsumption", "T", "a",
+				"aabd",
+				"[@2,2:2='b',<1>,1:2]\n",
+				"line 1:1 extraneous input 'a' expecting {'b', 'c'}\n");
 		file.addParserTest(input, "SingleTokenInsertion", "T", "a",
 				"ac",
 				"",
@@ -597,6 +601,10 @@ private JUnitTestFile buildParserErrors() throws Exception {
 				"ad",
 				"",
 				"line 1:1 missing {'b', 'c'} at 'd'\n");
+		file.addParserTest(input, "SingleSetInsertionConsumption", "T", "a",
+				"ad",
+				"[@0,0:0='a',<3>,1:0]\n",
+				"line 1:1 missing {'b', 'c'} at 'd'\n");
 		file.addParserTest(input, "ConjuringUpTokenFromSet", "T", "a",
 				"ad",
 				"conjured=[@-1,-1:-1='<missing 'b'>',<2>,1:1]\n",

tool/test/org/antlr/v4/test/rt/gen/grammars/ParserErrors/SingleSetInsertionConsumption.st

@@ -0,0 +1,3 @@
+grammar <grammarName>;
+set: ('b'|'c') ;
+a: 'a' set 'd' {System.out.println($set.stop);} ;

tool/test/org/antlr/v4/test/rt/gen/grammars/ParserErrors/SingleTokenDeletionConsumption.st

@@ -0,0 +1,3 @@
+grammar <grammarName>;
+set: ('b'|'c') ;
+a: 'a' set 'd' {System.out.println($set.stop);} ;

tool/test/org/antlr/v4/test/rt/java/TestParserErrors.java

@@ -35,6 +35,17 @@ public void testSingleTokenDeletionExpectingSet() throws Exception {
 		assertEquals("line 1:1 extraneous input 'a' expecting {'b', 'c'}\n", this.stderrDuringParse);
 	}
 
+	/* this file and method are generated, any edit will be overwritten by the next generation */
+	@Test
+	public void testSingleTokenDeletionConsumption() throws Exception {
+		String grammar = "grammar T;\n" +
+	                  "set: ('b'|'c') ;\n" +
+	                  "a: 'a' set 'd' {System.out.println($set.stop);} ;";
+		String found = execParser("T.g4", grammar, "TParser", "TLexer", "a", "aabd", false);
+		assertEquals("[@2,2:2='b',<1>,1:2]\n", found);
+		assertEquals("line 1:1 extraneous input 'a' expecting {'b', 'c'}\n", this.stderrDuringParse);
+	}
+
 	/* this file and method are generated, any edit will be overwritten by the next generation */
 	@Test
 	public void testSingleTokenInsertion() throws Exception {
@@ -65,6 +76,17 @@ public void testSingleSetInsertion() throws Exception {
 		assertEquals("line 1:1 missing {'b', 'c'} at 'd'\n", this.stderrDuringParse);
 	}
 
+	/* this file and method are generated, any edit will be overwritten by the next generation */
+	@Test
+	public void testSingleSetInsertionConsumption() throws Exception {
+		String grammar = "grammar T;\n" +
+	                  "set: ('b'|'c') ;\n" +
+	                  "a: 'a' set 'd' {System.out.println($set.stop);} ;";
+		String found = execParser("T.g4", grammar, "TParser", "TLexer", "a", "ad", false);
+		assertEquals("[@0,0:0='a',<3>,1:0]\n", found);
+		assertEquals("line 1:1 missing {'b', 'c'} at 'd'\n", this.stderrDuringParse);
+	}
+
 	/* this file and method are generated, any edit will be overwritten by the next generation */
 	@Test
 	public void testConjuringUpTokenFromSet() throws Exception {