Skip to content

Commit

Permalink
[Lex] Introduce Preprocessor::LexTokensUntilEOF()
Browse files Browse the repository at this point in the history 
This new method repeatedly calls Lex() until end of file is reached
and optionally fills a std::vector of Tokens. Use it in Clang's unit
tests to avoid quite some code duplication.

Differential Revision: https://reviews.llvm.org/D158413
  • Loading branch information
@hahnjo
hahnjo committed Oct 5, 2023
1 parent 7fa3377 commit 3116d60
Show file tree
Hide file tree
Showing 10 changed files with 30 additions and 104 deletions.
3 changes: 3 additions & 0 deletions clang/include/clang/Lex/Preprocessor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1722,6 +1722,9 @@ class Preprocessor {
/// Lex the next token for this preprocessor.
void Lex(Token &Result);

/// Lex all tokens for this preprocessor until (and excluding) end of file.
void LexTokensUntilEOF(std::vector<Token> *Tokens = nullptr);

/// Lex a token, forming a header-name token if possible.
bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);

Expand Down
11 changes: 11 additions & 0 deletions clang/lib/Lex/Preprocessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -998,6 +998,17 @@ void Preprocessor::Lex(Token &Result) {
}
}

void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) {
while (1) {
Token Tok;
Lex(Tok);
if (Tok.isOneOf(tok::unknown, tok::eof, tok::eod))
break;
if (Tokens != nullptr)
Tokens->push_back(Tok);
}
}

/// Lex a header-name token (including one formed from header-name-tokens if
/// \p AllowConcatenation is \c true).
///
Expand Down
7 changes: 1 addition & 6 deletions clang/unittests/Analysis/MacroExpansionContextTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,12 +73,7 @@ class MacroExpansionContextTest : public ::testing::Test {
// Lex source text.
PP.EnterMainSourceFile();

while (true) {
Token Tok;
PP.Lex(Tok);
if (Tok.is(tok::eof))
break;
}
PP.LexTokensUntilEOF();

// Callbacks have been executed at this point.
return Ctx;
Expand Down
32 changes: 4 additions & 28 deletions clang/unittests/Basic/SourceManagerTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,13 +138,7 @@ TEST_F(SourceManagerTest, isBeforeInTranslationUnit) {
PP.EnterMainSourceFile();

std::vector<Token> toks;
while (1) {
Token tok;
PP.Lex(tok);
if (tok.is(tok::eof))
break;
toks.push_back(tok);
}
PP.LexTokensUntilEOF(&toks);

// Make sure we got the tokens that we expected.
ASSERT_EQ(3U, toks.size());
Expand Down Expand Up @@ -195,13 +189,7 @@ TEST_F(SourceManagerTest, isBeforeInTranslationUnitWithTokenSplit) {
llvm::SmallString<8> Scratch;

std::vector<Token> toks;
while (1) {
Token tok;
PP.Lex(tok);
if (tok.is(tok::eof))
break;
toks.push_back(tok);
}
PP.LexTokensUntilEOF(&toks);

// Make sure we got the tokens that we expected.
ASSERT_EQ(4U, toks.size()) << "a >> b c";
Expand Down Expand Up @@ -452,13 +440,7 @@ TEST_F(SourceManagerTest, getMacroArgExpandedLocation) {
PP.EnterMainSourceFile();

std::vector<Token> toks;
while (1) {
Token tok;
PP.Lex(tok);
if (tok.is(tok::eof))
break;
toks.push_back(tok);
}
PP.LexTokensUntilEOF(&toks);

// Make sure we got the tokens that we expected.
ASSERT_EQ(4U, toks.size());
Expand Down Expand Up @@ -574,13 +556,7 @@ TEST_F(SourceManagerTest, isBeforeInTranslationUnitWithMacroInInclude) {
PP.EnterMainSourceFile();

std::vector<Token> toks;
while (1) {
Token tok;
PP.Lex(tok);
if (tok.is(tok::eof))
break;
toks.push_back(tok);
}
PP.LexTokensUntilEOF(&toks);

// Make sure we got the tokens that we expected.
ASSERT_EQ(0U, toks.size());
Expand Down
15 changes: 2 additions & 13 deletions clang/unittests/Lex/LexerTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,13 +74,7 @@ class LexerTest : public ::testing::Test {
PP = CreatePP(Source, ModLoader);

std::vector<Token> toks;
while (1) {
Token tok;
PP->Lex(tok);
if (tok.is(tok::eof))
break;
toks.push_back(tok);
}
PP->LexTokensUntilEOF(&toks);

return toks;
}
Expand Down Expand Up @@ -628,12 +622,7 @@ TEST_F(LexerTest, FindNextToken) {
TEST_F(LexerTest, CreatedFIDCountForPredefinedBuffer) {
TrivialModuleLoader ModLoader;
auto PP = CreatePP("", ModLoader);
while (1) {
Token tok;
PP->Lex(tok);
if (tok.is(tok::eof))
break;
}
PP->LexTokensUntilEOF();
EXPECT_EQ(SourceMgr.getNumCreatedFIDsForFileID(PP->getPredefinesFileID()),
1U);
}
Expand Down
7 changes: 1 addition & 6 deletions clang/unittests/Lex/ModuleDeclStateTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,7 @@ class ModuleDeclStateTest : public ::testing::Test {
PP.addPPCallbacks(std::move(C));
PP.EnterMainSourceFile();

while (1) {
Token tok;
PP.Lex(tok);
if (tok.is(tok::eof))
break;
}
PP.LexTokensUntilEOF();
}

FileSystemOptions FileMgrOpts;
Expand Down
37 changes: 5 additions & 32 deletions clang/unittests/Lex/PPCallbacksTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,13 +229,7 @@ class PPCallbacksTest : public ::testing::Test {

// Lex source text.
PP.EnterMainSourceFile();

while (true) {
Token Tok;
PP.Lex(Tok);
if (Tok.is(tok::eof))
break;
}
PP.LexTokensUntilEOF();

// Callbacks have been executed at this point -- return filename range.
return Callbacks;
Expand All @@ -259,13 +253,7 @@ class PPCallbacksTest : public ::testing::Test {

// Lex source text.
PP.EnterMainSourceFile();

while (true) {
Token Tok;
PP.Lex(Tok);
if (Tok.is(tok::eof))
break;
}
PP.LexTokensUntilEOF();

return Callbacks->Results;
}
Expand All @@ -290,12 +278,7 @@ class PPCallbacksTest : public ::testing::Test {

// Lex source text.
PP.EnterMainSourceFile();
while (true) {
Token Tok;
PP.Lex(Tok);
if (Tok.is(tok::eof))
break;
}
PP.LexTokensUntilEOF();

return Callbacks->Marks;
}
Expand Down Expand Up @@ -334,12 +317,7 @@ class PPCallbacksTest : public ::testing::Test {

// Lex source text.
PP.EnterMainSourceFile();
while (true) {
Token Tok;
PP.Lex(Tok);
if (Tok.is(tok::eof))
break;
}
PP.LexTokensUntilEOF();

PragmaOpenCLExtensionCallbacks::CallbackParameters RetVal = {
Callbacks->Name,
Expand Down Expand Up @@ -477,12 +455,7 @@ TEST_F(PPCallbacksTest, FileNotFoundSkipped) {

// Lex source text.
PP.EnterMainSourceFile();
while (true) {
Token Tok;
PP.Lex(Tok);
if (Tok.is(tok::eof))
break;
}
PP.LexTokensUntilEOF();

ASSERT_EQ(1u, Callbacks->NumCalls);
ASSERT_EQ(0u, DiagConsumer->getNumErrors());
Expand Down
8 changes: 1 addition & 7 deletions clang/unittests/Lex/PPConditionalDirectiveRecordTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,7 @@ TEST_F(PPConditionalDirectiveRecordTest, PPRecAPI) {
PP.EnterMainSourceFile();

std::vector<Token> toks;
while (1) {
Token tok;
PP.Lex(tok);
if (tok.is(tok::eof))
break;
toks.push_back(tok);
}
PP.LexTokensUntilEOF(&toks);

// Make sure we got the tokens that we expected.
ASSERT_EQ(10U, toks.size());
Expand Down
7 changes: 1 addition & 6 deletions clang/unittests/Lex/PPDependencyDirectivesTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,12 +133,7 @@ TEST_F(PPDependencyDirectivesTest, MacroGuard) {
SmallVector<StringRef> IncludedFiles;
PP.addPPCallbacks(std::make_unique<IncludeCollector>(PP, IncludedFiles));
PP.EnterMainSourceFile();
while (true) {
Token tok;
PP.Lex(tok);
if (tok.is(tok::eof))
break;
}
PP.LexTokensUntilEOF();

SmallVector<std::string> IncludedFilesSlash;
for (StringRef IncludedFile : IncludedFiles)
Expand Down
7 changes: 1 addition & 6 deletions clang/unittests/Lex/PPMemoryAllocationsTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,12 +75,7 @@ TEST_F(PPMemoryAllocationsTest, PPMacroDefinesAllocations) {
PP.Initialize(*Target);
PP.EnterMainSourceFile();

while (1) {
Token tok;
PP.Lex(tok);
if (tok.is(tok::eof))
break;
}
PP.LexTokensUntilEOF();

size_t NumAllocated = PP.getPreprocessorAllocator().getBytesAllocated();
float BytesPerDefine = float(NumAllocated) / float(NumMacros);
Expand Down

0 comments on commit 3116d60

Please sign in to comment.