diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp index 25ac9be57c81a..52a55ea23b5f2 100644 --- a/clang/lib/Format/FormatTokenLexer.cpp +++ b/clang/lib/Format/FormatTokenLexer.cpp @@ -93,8 +93,10 @@ ArrayRef FormatTokenLexer::lex() { // string literals are correctly identified. handleCSharpVerbatimAndInterpolatedStrings(); } - if (Style.isTableGen()) + if (Style.isTableGen()) { handleTableGenMultilineString(); + handleTableGenNumericLikeIdentifier(); + } if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline) FirstInLineIndex = Tokens.size() - 1; } while (Tokens.back()->isNot(tok::eof)); @@ -804,6 +806,44 @@ void FormatTokenLexer::handleTableGenMultilineString() { FirstLineText, MultiLineString->OriginalColumn, Style.TabWidth, Encoding); } +void FormatTokenLexer::handleTableGenNumericLikeIdentifier() { + FormatToken *Tok = Tokens.back(); + // TableGen identifiers can begin with digits. Such tokens are lexed as + // numeric_constant now. + if (Tok->isNot(tok::numeric_constant)) + return; + StringRef Text = Tok->TokenText; + // The following check is based on llvm::TGLexer::LexToken. + // That lexes the token as a number if any of the following holds: + // 1. It starts with '+', '-'. + // 2. All the characters are digits. + // 3. The first non-digit character is 'b', and the next is '0' or '1'. + // 4. The first non-digit character is 'x', and the next is a hex digit. + // Note that in the case 3 and 4, if the next character does not exists in + // this token, the token is an identifier. + if (Text.size() < 1 || Text[0] == '+' || Text[0] == '-') + return; + const auto NonDigitPos = Text.find_if([](char C) { return !isdigit(C); }); + // All the characters are digits + if (NonDigitPos == StringRef::npos) + return; + char FirstNonDigit = Text[NonDigitPos]; + if (NonDigitPos < Text.size() - 1) { + char TheNext = Text[NonDigitPos + 1]; + // Regarded as a binary number. + if (FirstNonDigit == 'b' && (TheNext == '0' || TheNext == '1')) + return; + // Regarded as hex number. + if (FirstNonDigit == 'x' && isxdigit(TheNext)) + return; + } + if (isalpha(FirstNonDigit) || FirstNonDigit == '_') { + // This is actually an identifier in TableGen. + Tok->Tok.setKind(tok::identifier); + Tok->Tok.setIdentifierInfo(nullptr); + } +} + void FormatTokenLexer::handleTemplateStrings() { FormatToken *BacktickToken = Tokens.back(); diff --git a/clang/lib/Format/FormatTokenLexer.h b/clang/lib/Format/FormatTokenLexer.h index 1dec6bbc41514..65dd733bd5335 100644 --- a/clang/lib/Format/FormatTokenLexer.h +++ b/clang/lib/Format/FormatTokenLexer.h @@ -97,6 +97,10 @@ class FormatTokenLexer { // Handles TableGen multiline strings. It has the form [{ ... }]. void handleTableGenMultilineString(); + // Handles TableGen numeric like identifiers. + // They have a forms of [0-9]*[_a-zA-Z]([_a-zA-Z0-9]*). But limited to the + // case it is not lexed as an integer. + void handleTableGenNumericLikeIdentifier(); void tryParsePythonComment(); diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index 117d8fe8f7dc1..64b2abac5cce5 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -2209,6 +2209,29 @@ TEST_F(TokenAnnotatorTest, UnderstandTableGenTokens) { EXPECT_EQ(Tokens[0]->ColumnWidth, sizeof("[{ It can break\n") - 1); EXPECT_TRUE(Tokens[0]->IsMultiline); EXPECT_EQ(Tokens[0]->LastLineColumnWidth, sizeof(" the string. }]") - 1); + + // Identifier tokens. In TableGen, identifiers can begin with a number. + // In ambiguous cases, the lexer tries to lex it as a number. + // Even if the try fails, it does not fall back to identifier lexing and + // regard as an error. + // The ambiguity is not documented. The result of those tests are based on the + // implementation of llvm::TGLexer::LexToken. + Tokens = Annotate("1234"); + EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown); + Tokens = Annotate("0x1abC"); + EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown); + // This is invalid syntax of number, but not an identifier. + Tokens = Annotate("0x1234x"); + EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown); + Tokens = Annotate("identifier"); + EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown); + // Identifier beginning with a number. + Tokens = Annotate("0x"); + EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown); + Tokens = Annotate("2dVector"); + EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown); + Tokens = Annotate("01234Vector"); + EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown); } TEST_F(TokenAnnotatorTest, UnderstandConstructors) {