Skip to content

Commit

Permalink
Perf/lexer faster slow get char and size (#70543)
Browse files Browse the repository at this point in the history 
Co-authored-by: serge-sans-paille <sguelton@mozilla.com>
  • Loading branch information
@serge-sans-paille
serge-sans-paille and serge-sans-paille committed Oct 29, 2023
1 parent 31b9121 commit d8f5a18
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 53 deletions.
35 changes: 18 additions & 17 deletions clang/include/clang/Lex/Lexer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -575,19 +575,23 @@ class Lexer : public PreprocessorLexer {
/// sequence.
static bool isNewLineEscaped(const char *BufferStart, const char *Str);

/// Represents a char and the number of bytes parsed to produce it.
struct SizedChar {
char Char;
unsigned Size;
};

/// getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever
/// emit a warning.
static inline char getCharAndSizeNoWarn(const char *Ptr, unsigned &Size,
const LangOptions &LangOpts) {
static inline SizedChar getCharAndSizeNoWarn(const char *Ptr,
const LangOptions &LangOpts) {
// If this is not a trigraph and not a UCN or escaped newline, return
// quickly.
if (isObviouslySimpleCharacter(Ptr[0])) {
Size = 1;
return *Ptr;
return {*Ptr, 1u};
}

Size = 0;
return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts);
return getCharAndSizeSlowNoWarn(Ptr, LangOpts);
}

/// Returns the leading whitespace for line that corresponds to the given
Expand Down Expand Up @@ -665,8 +669,7 @@ class Lexer : public PreprocessorLexer {
// quickly.
if (isObviouslySimpleCharacter(Ptr[0])) return *Ptr++;

unsigned Size = 0;
char C = getCharAndSizeSlow(Ptr, Size, &Tok);
auto [C, Size] = getCharAndSizeSlow(Ptr, &Tok);
Ptr += Size;
return C;
}
Expand All @@ -682,9 +685,7 @@ class Lexer : public PreprocessorLexer {

// Otherwise, re-lex the character with a current token, allowing
// diagnostics to be emitted and flags to be set.
Size = 0;
getCharAndSizeSlow(Ptr, Size, &Tok);
return Ptr+Size;
return Ptr + getCharAndSizeSlow(Ptr, &Tok).Size;
}

/// getCharAndSize - Peek a single 'character' from the specified buffer,
Expand All @@ -699,14 +700,14 @@ class Lexer : public PreprocessorLexer {
return *Ptr;
}

Size = 0;
return getCharAndSizeSlow(Ptr, Size);
auto CharAndSize = getCharAndSizeSlow(Ptr);
Size = CharAndSize.Size;
return CharAndSize.Char;
}

/// getCharAndSizeSlow - Handle the slow/uncommon case of the getCharAndSize
/// method.
char getCharAndSizeSlow(const char *Ptr, unsigned &Size,
Token *Tok = nullptr);
SizedChar getCharAndSizeSlow(const char *Ptr, Token *Tok = nullptr);

/// getEscapedNewLineSize - Return the size of the specified escaped newline,
/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" on entry
Expand All @@ -720,8 +721,8 @@ class Lexer : public PreprocessorLexer {

/// getCharAndSizeSlowNoWarn - Same as getCharAndSizeSlow, but never emits a
/// diagnostic.
static char getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
const LangOptions &LangOpts);
static SizedChar getCharAndSizeSlowNoWarn(const char *Ptr,
const LangOptions &LangOpts);

//===--------------------------------------------------------------------===//
// Other lexer functions.
Expand Down
5 changes: 2 additions & 3 deletions clang/lib/Lex/DependencyDirectivesScanner.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -565,9 +565,8 @@ Scanner::cleanStringIfNeeded(const dependency_directives_scan::Token &Tok) {
const char *BufPtr = Input.begin() + Tok.Offset;
const char *AfterIdent = Input.begin() + Tok.getEnd();
while (BufPtr < AfterIdent) {
unsigned Size;
Spelling[SpellingLength++] =
Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
auto [Char, Size] = Lexer::getCharAndSizeNoWarn(BufPtr, LangOpts);
Spelling[SpellingLength++] = Char;
BufPtr += Size;
}

Expand Down
73 changes: 40 additions & 33 deletions clang/lib/Lex/Lexer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,9 +287,9 @@ static size_t getSpellingSlow(const Token &Tok, const char *BufPtr,
if (tok::isStringLiteral(Tok.getKind())) {
// Munch the encoding-prefix and opening double-quote.
while (BufPtr < BufEnd) {
unsigned Size;
Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
BufPtr += Size;
auto CharAndSize = Lexer::getCharAndSizeNoWarn(BufPtr, LangOpts);
Spelling[Length++] = CharAndSize.Char;
BufPtr += CharAndSize.Size;

if (Spelling[Length - 1] == '"')
break;
Expand All @@ -316,9 +316,9 @@ static size_t getSpellingSlow(const Token &Tok, const char *BufPtr,
}

while (BufPtr < BufEnd) {
unsigned Size;
Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
BufPtr += Size;
auto CharAndSize = Lexer::getCharAndSizeNoWarn(BufPtr, LangOpts);
Spelling[Length++] = CharAndSize.Char;
BufPtr += CharAndSize.Size;
}

assert(Length < Tok.getLength() &&
Expand Down Expand Up @@ -772,10 +772,9 @@ unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo,
// If we have a character that may be a trigraph or escaped newline, use a
// lexer to parse it correctly.
for (; CharNo; --CharNo) {
unsigned Size;
Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts);
TokPtr += Size;
PhysOffset += Size;
auto CharAndSize = Lexer::getCharAndSizeNoWarn(TokPtr, LangOpts);
TokPtr += CharAndSize.Size;
PhysOffset += CharAndSize.Size;
}

// Final detail: if we end up on an escaped newline, we want to return the
Expand Down Expand Up @@ -1357,15 +1356,16 @@ SourceLocation Lexer::findLocationAfterToken(
///
/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
/// be updated to match.
char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
Token *Tok) {
Lexer::SizedChar Lexer::getCharAndSizeSlow(const char *Ptr, Token *Tok) {
unsigned Size = 0;
// If we have a slash, look for an escaped newline.
if (Ptr[0] == '\\') {
++Size;
++Ptr;
Slash:
// Common case, backslash-char where the char is not whitespace.
if (!isWhitespace(Ptr[0])) return '\\';
if (!isWhitespace(Ptr[0]))
return {'\\', Size};

// See if we have optional whitespace characters between the slash and
// newline.
Expand All @@ -1382,11 +1382,13 @@ char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
Ptr += EscapedNewLineSize;

// Use slow version to accumulate a correct size field.
return getCharAndSizeSlow(Ptr, Size, Tok);
auto CharAndSize = getCharAndSizeSlow(Ptr, Tok);
CharAndSize.Size += Size;
return CharAndSize;
}

// Otherwise, this is not an escaped newline, just return the slash.
return '\\';
return {'\\', Size};
}

// If this is a trigraph, process it.
Expand All @@ -1401,13 +1403,12 @@ char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
Ptr += 3;
Size += 3;
if (C == '\\') goto Slash;
return C;
return {C, Size};
}
}

// If this is neither, return a single character.
++Size;
return *Ptr;
return {*Ptr, Size + 1u};
}

/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
Expand All @@ -1416,15 +1417,18 @@ char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
///
/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
/// be updated to match.
char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
const LangOptions &LangOpts) {
Lexer::SizedChar Lexer::getCharAndSizeSlowNoWarn(const char *Ptr,
const LangOptions &LangOpts) {

unsigned Size = 0;
// If we have a slash, look for an escaped newline.
if (Ptr[0] == '\\') {
++Size;
++Ptr;
Slash:
// Common case, backslash-char where the char is not whitespace.
if (!isWhitespace(Ptr[0])) return '\\';
if (!isWhitespace(Ptr[0]))
return {'\\', Size};

// See if we have optional whitespace characters followed by a newline.
if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
Expand All @@ -1433,11 +1437,13 @@ char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
Ptr += EscapedNewLineSize;

// Use slow version to accumulate a correct size field.
return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts);
auto CharAndSize = getCharAndSizeSlowNoWarn(Ptr, LangOpts);
CharAndSize.Size += Size;
return CharAndSize;
}

// Otherwise, this is not an escaped newline, just return the slash.
return '\\';
return {'\\', Size};
}

// If this is a trigraph, process it.
Expand All @@ -1448,13 +1454,12 @@ char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
Ptr += 3;
Size += 3;
if (C == '\\') goto Slash;
return C;
return {C, Size};
}
}

// If this is neither, return a single character.
++Size;
return *Ptr;
return {*Ptr, Size + 1u};
}

//===----------------------------------------------------------------------===//
Expand Down Expand Up @@ -1964,11 +1969,14 @@ bool Lexer::LexIdentifierContinue(Token &Result, const char *CurPtr) {
/// isHexaLiteral - Return true if Start points to a hex constant.
/// in microsoft mode (where this is supposed to be several different tokens).
bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
unsigned Size;
char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts);
auto CharAndSize1 = Lexer::getCharAndSizeNoWarn(Start, LangOpts);
char C1 = CharAndSize1.Char;
if (C1 != '0')
return false;
char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts);

auto CharAndSize2 =
Lexer::getCharAndSizeNoWarn(Start + CharAndSize1.Size, LangOpts);
char C2 = CharAndSize2.Char;
return (C2 == 'x' || C2 == 'X');
}

Expand Down Expand Up @@ -2012,8 +2020,7 @@ bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {

// If we have a digit separator, continue.
if (C == '\'' && (LangOpts.CPlusPlus14 || LangOpts.C23)) {
unsigned NextSize;
char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, LangOpts);
auto [Next, NextSize] = getCharAndSizeNoWarn(CurPtr + Size, LangOpts);
if (isAsciiIdentifierContinue(Next)) {
if (!isLexingRawMode())
Diag(CurPtr, LangOpts.CPlusPlus
Expand Down Expand Up @@ -2085,8 +2092,8 @@ const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr,
unsigned Consumed = Size;
unsigned Chars = 1;
while (true) {
unsigned NextSize;
char Next = getCharAndSizeNoWarn(CurPtr + Consumed, NextSize, LangOpts);
auto [Next, NextSize] =
getCharAndSizeNoWarn(CurPtr + Consumed, LangOpts);
if (!isAsciiIdentifierContinue(Next)) {
// End of suffix. Check whether this is on the allowed list.
const StringRef CompleteSuffix(Buffer, Chars);
Expand Down

0 comments on commit d8f5a18

Please sign in to comment.