diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 6e6ec0969b54c6..1a19b9bbbdb520 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -34,6 +34,8 @@ under the License. ${basedir}/../../ ${basedir}/../../thirdparty 1 + + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 83f978fff948b9..2ce88a9c541a82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -56,7 +56,6 @@ import org.apache.logging.log4j.Logger; import java.lang.reflect.Method; -import java.util.BitSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -73,22 +72,10 @@ public class NereidsParser { private static final ParseErrorListener PARSE_ERROR_LISTENER = new ParseErrorListener(); private static final PostProcessor POST_PROCESSOR = new PostProcessor(); - private static final BitSet EXPLAIN_TOKENS = new BitSet(); - private static final Set NON_RESERVED_KEYWORDS; private static final Map LITERAL_TOKENS; static { - EXPLAIN_TOKENS.set(DorisLexer.EXPLAIN); - EXPLAIN_TOKENS.set(DorisLexer.PARSED); - EXPLAIN_TOKENS.set(DorisLexer.ANALYZED); - EXPLAIN_TOKENS.set(DorisLexer.LOGICAL); - EXPLAIN_TOKENS.set(DorisLexer.REWRITTEN); - EXPLAIN_TOKENS.set(DorisLexer.PHYSICAL); - EXPLAIN_TOKENS.set(DorisLexer.OPTIMIZED); - EXPLAIN_TOKENS.set(DorisLexer.PLAN); - EXPLAIN_TOKENS.set(DorisLexer.PROCESS); - ImmutableSet.Builder nonReserveds = ImmutableSet.builder(); for (Method declaredMethod : NonReservedContext.class.getDeclaredMethods()) { if (TerminalNode.class.equals(declaredMethod.getReturnType()) @@ -414,13 +401,14 @@ public static ParserRuleContext toAst( ParserRuleContext tree; try { // first, try parsing with potentially faster SLL mode + parser.setErrorHandler(new ParserBailErrorStrategy()); parser.getInterpreter().setPredictionMode(PredictionMode.SLL); tree = parseFunction.apply(parser); } catch (ParseCancellationException ex) { // if we fail, parse with LL mode tokenStream.seek(0); // rewind input stream parser.reset(); - + parser.setErrorHandler(new ParseErrorStrategy()); parser.getInterpreter().setPredictionMode(PredictionMode.LL); tree = parseFunction.apply(parser); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParseErrorStrategy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParseErrorStrategy.java new file mode 100644 index 00000000000000..b97f6704e94978 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParseErrorStrategy.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import com.google.common.collect.ImmutableMap; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.InputMismatchException; +import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Token; +import org.apache.commons.lang3.StringUtils; + +import java.util.Map; + +/** + * A ParserErrorStrategy extends the {@link DefaultErrorStrategy}, that does special handling + * on errors. + * + * The intention of this class is to provide more information of these errors encountered in ANTLR + * parser to the downstream consumers. + */ +public class ParseErrorStrategy extends DefaultErrorStrategy { + private static final Map USER_WORD_DICT = ImmutableMap.of("''", "end of input"); + + @Override + protected String getTokenErrorDisplay(Token t) { + String tokenName = super.getTokenErrorDisplay(t); + return USER_WORD_DICT.getOrDefault(tokenName, tokenName); + } + + @Override + protected void reportInputMismatch(Parser recognizer, InputMismatchException e) { + recognizer.notifyErrorListeners(e.getOffendingToken(), + constructErrorMsg(e.getOffendingToken(), ""), e); + } + + @Override + protected void reportNoViableAlternative(Parser recognizer, NoViableAltException e) { + recognizer.notifyErrorListeners(e.getOffendingToken(), + constructErrorMsg(e.getOffendingToken(), ""), e); + } + + @Override + protected void reportUnwantedToken(Parser recognizer) { + if (this.inErrorRecoveryMode(recognizer)) { + return; + } + this.beginErrorCondition(recognizer); + recognizer.notifyErrorListeners(recognizer.getCurrentToken(), + constructErrorMsg(recognizer.getCurrentToken(), + "extra input " + getTokenErrorDisplay(recognizer.getCurrentToken())), + new RecognitionException(null, null, null)); + } + + @Override + protected void reportMissingToken(Parser recognizer) { + if (this.inErrorRecoveryMode(recognizer)) { + return; + } + this.beginErrorCondition(recognizer); + recognizer.notifyErrorListeners(recognizer.getCurrentToken(), + constructErrorMsg(recognizer.getCurrentToken(), + "missing " + getExpectedTokens(recognizer).toString(recognizer.getVocabulary())), + new RecognitionException(null, null, null)); + + } + + private String constructErrorMsg(Token token, String extraMessage) { + StringBuilder messageBuilder = new StringBuilder(); + messageBuilder.append("Syntax error at or near "); + messageBuilder.append(getTokenErrorDisplay(token)); + if (StringUtils.isNotEmpty(extraMessage)) { + messageBuilder.append(": "); + messageBuilder.append(extraMessage); + } + return messageBuilder.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParserBailErrorStrategy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParserBailErrorStrategy.java new file mode 100644 index 00000000000000..575ca0d0831884 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParserBailErrorStrategy.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.antlr.v4.runtime.InputMismatchException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.misc.ParseCancellationException; + +/** + * Inspired by {@link org.antlr.v4.runtime.BailErrorStrategy}, which is used in two-stage parsing: + * This error strategy allows the first stage of two-stage parsing to immediately terminate if an + * error is encountered, and immediately fall back to the second stage. In addition to avoiding + * wasted work by attempting to recover from errors here, the empty implementation of sync + * improves the performance of the first stage. + */ +public class ParserBailErrorStrategy extends ParseErrorStrategy { + /** + * Instead of recovering from exception e, re-throw it wrapped in a + * {@link ParseCancellationException} so it is not caught by the rule function catches. Use + * {@link Exception#getCause} to get the original {@link RecognitionException}. + */ + @Override + public void recover(Parser recognizer, RecognitionException e) { + ParserRuleContext context = recognizer.getContext(); + while (context != null) { + context.exception = e; + context = context.getParent(); + } + throw new ParseCancellationException(); + } + + /** + * Make sure we don't attempt to recover inline; if the parser successfully recovers, it won't + * throw an exception. + */ + @Override + public Token recoverInline(Parser recognizer) throws RecognitionException { + InputMismatchException e = new InputMismatchException(recognizer); + ParserRuleContext context = recognizer.getContext(); + while (context != null) { + context.exception = e; + context = context.getParent(); + } + throw new ParseCancellationException(e); + } + + /** Make sure we don't attempt to recover from problems in subrules. */ + @Override + public void sync(Parser recognizer) throws RecognitionException { + + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java index 41017520046151..c75c7c3f73c924 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java @@ -148,9 +148,7 @@ public void testExprArithmetic() { String subtract = "3 - 2"; assertExpr(subtract); - parseExpression("3 += 2") - .assertThrowsExactly(SyntaxParseException.class) - .assertMessageContains("extraneous input '=' expecting {'("); + parseExpression("3 += 2").assertThrowsExactly(SyntaxParseException.class); } diff --git a/regression-test/suites/doc/sql-manual/sql-functions/doc_date_error.groovy b/regression-test/suites/doc/sql-manual/sql-functions/doc_date_error.groovy index 866a8879cd50f9..83dc71aeb06938 100644 --- a/regression-test/suites/doc/sql-manual/sql-functions/doc_date_error.groovy +++ b/regression-test/suites/doc/sql-manual/sql-functions/doc_date_error.groovy @@ -37,7 +37,7 @@ suite("doc_date_error") { } test { sql """select DATE_ADD('2023-12-31 23:00:00', INTERVAL 2 sa);""" - exception "mismatched input 'sa' expecting" + exception "mismatched input 'sa'" } // date_ceil out of range diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy index 4bf1d347950ec4..7b736641d2f449 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy @@ -148,7 +148,7 @@ suite("iceberg_branch_tag_operate", "p0,external") { // Test CREATE OR REPLACE BRANCH IF NOT EXISTS - this should fail test { sql """ alter table ${table_name} create or replace branch if not exists b7 """ - exception "mismatched input" + exception "Syntax error near 'if'" } sql """ alter table ${table_name} create or replace branch b7 """