Skip to content

testrunner: some SimpleTokenizer refactoring #7467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chrchr-github merged 4 commits into from
Apr 15, 2025
Merged

testrunner: some SimpleTokenizer refactoring #7467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ba9648c
helpers.h: removed some unused `SimpleTokenizer` code
firewave Apr 15, 2025
8bb1416
testrunner: removed duplicated language parameter from `SimpleTokenizer`
firewave Apr 15, 2025
b48d86d
testrunner: moved `tokenize()` call out of a `SimpleTokenizer` constr…
firewave Apr 15, 2025
612fbad
testrunner: got rid of `SimpleTokenizer::tokenize()` overload with fi…
firewave Apr 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 15 additions & 41 deletions test/helpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,81 +44,55 @@ namespace tinyxml2 {
// TODO: make Tokenizer private
class SimpleTokenizer : public Tokenizer {
public:
template<size_t size>
SimpleTokenizer(ErrorLogger& errorlogger, const char (&code)[size], bool cpp = true)
explicit SimpleTokenizer(ErrorLogger& errorlogger, bool cpp = true)
: Tokenizer{s_settings, errorlogger}
{
if (!tokenize(code, cpp))
throw std::runtime_error("creating tokens failed");
list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true);
}

SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger)
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, bool cpp = true)
: Tokenizer{settings, errorlogger}
{}

/*
Token* tokens() {
return Tokenizer::tokens();
}

const Token* tokens() const {
return Tokenizer::tokens();
}
*/

template<size_t size>
bool tokenize(const char (&code)[size],
const std::string& filename,
const std::string &configuration = "")
{
std::istringstream istr(code);
return tokenize(istr, filename, configuration);
list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true);
}

template<size_t size>
bool tokenize(const char (&code)[size],
bool cpp = true,
const std::string &configuration = "")
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const std::string& filename)
: Tokenizer{settings, errorlogger}
{
std::istringstream istr(code);
return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c"), configuration);
list.setLang(Path::identify(filename, false));
list.appendFileIfNew(filename);
}

bool tokenize(const std::string& code,
const std::string& filename,
const std::string &configuration = "")
template<size_t size>
bool tokenize(const char (&code)[size])
{
std::istringstream istr(code);
return tokenize(istr, filename, configuration);
return tokenize(istr, std::string(list.isCPP() ? "test.cpp" : "test.c"));
}

bool tokenize(const std::string& code,
bool cpp = true,
const std::string &configuration = "")
bool tokenize(const std::string& code)
{
std::istringstream istr(code);
return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c"), configuration);
return tokenize(istr, std::string(list.isCPP() ? "test.cpp" : "test.c"));
}

private:
/**
* Tokenize code
* @param istr The code as stream
* @param filename Indicates if the code is C++
* @param configuration E.g. "A" for code where "#ifdef A" is true
* @return false if source code contains syntax errors
*/
bool tokenize(std::istream& istr,
const std::string& filename,
const std::string &configuration = "")
const std::string& filename)
{
if (list.front())
throw std::runtime_error("token list is not empty");
list.appendFileIfNew(filename);
if (!list.createTokens(istr, Path::identify(filename, false)))
return false;

return simplifyTokens1(configuration);
return simplifyTokens1("");
}

// TODO: find a better solution
Expand Down
4 changes: 2 additions & 2 deletions test/test64bit.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ class Test64BitPortability : public TestFixture {
template<size_t size>
void check_(const char* file, int line, const char (&code)[size], bool cpp = true) {
// Tokenize..
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
SimpleTokenizer tokenizer(settings, *this, cpp);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check char variable usage..
Check64BitPortability check64BitPortability(&tokenizer, &settings, this);
Expand Down
4 changes: 2 additions & 2 deletions test/testastutils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,8 @@ class TestAstUtils : public TestFixture {
#define isSameExpression(...) isSameExpression_(__FILE__, __LINE__, __VA_ARGS__)
template<size_t size>
bool isSameExpression_(const char* file, int line, const char (&code)[size], const char tokStr1[], const char tokStr2[], bool cpp) {
SimpleTokenizer tokenizer(settingsDefault, *this);
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
SimpleTokenizer tokenizer(settingsDefault, *this, cpp);
ASSERT_LOC(tokenizer.tokenize(code), file, line);
const Token * const tok1 = Token::findsimplematch(tokenizer.tokens(), tokStr1, strlen(tokStr1));
const Token * const tok2 = Token::findsimplematch(tok1->next(), tokStr2, strlen(tokStr2));
return (isSameExpression)(false, tok1, tok2, settingsDefault, false, true);
Expand Down
4 changes: 2 additions & 2 deletions test/testautovariables.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ class TestAutoVariables : public TestFixture {
const Settings settings1 = settingsBuilder(settings).certainty(Certainty::inconclusive, options.inconclusive).build();

// Tokenize..
SimpleTokenizer tokenizer(settings1, *this);
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
SimpleTokenizer tokenizer(settings1, *this, options.cpp);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

runChecks<CheckAutoVariables>(tokenizer, this);
}
Expand Down
4 changes: 2 additions & 2 deletions test/testbool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,8 @@ class TestBool : public TestFixture {
template<size_t size>
void check_(const char* file, int line, const char (&code)[size], const CheckOptions& options = make_default_obj()) {
// Tokenize..
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
SimpleTokenizer tokenizer(settings, *this, options.cpp);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check...
runChecks<CheckBool>(tokenizer, this);
Expand Down
4 changes: 2 additions & 2 deletions test/testbufferoverrun.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ class TestBufferOverrun : public TestFixture {
const Settings settings = options.s ? *options.s : settingsBuilder(settings0).certainty(Certainty::inconclusive).build();

// Tokenize..
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
SimpleTokenizer tokenizer(settings, *this, options.cpp);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check for buffer overruns..
runChecks<CheckBufferOverrun>(tokenizer, this);
Expand Down
4 changes: 2 additions & 2 deletions test/testclass.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9081,9 +9081,9 @@ class TestClass : public TestFixture {
// getFileInfo
std::list<Check::FileInfo*> fileInfo;
for (const std::string& c: code) {
SimpleTokenizer tokenizer{settingsDefault, *this};
const std::string filename = std::to_string(fileInfo.size()) + ".cpp";
ASSERT(tokenizer.tokenize(c, filename));
SimpleTokenizer tokenizer{settingsDefault, *this, filename};
ASSERT(tokenizer.tokenize(c));
fileInfo.push_back(check.getFileInfo(tokenizer, settingsDefault));
}

Expand Down
4 changes: 2 additions & 2 deletions test/testfunctions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,8 @@ class TestFunctions : public TestFixture {
const Settings& s = options.s ? *options.s : settings;

// Tokenize..
SimpleTokenizer tokenizer(s, *this);
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
SimpleTokenizer tokenizer(s, *this, options.cpp);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

runChecks<CheckFunctions>(tokenizer, this);
}
Expand Down
8 changes: 4 additions & 4 deletions test/testgarbage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,8 +288,8 @@ class TestGarbage : public TestFixture {
template<size_t size>
std::string checkCodeInternal_(const char* file, int line, const char (&code)[size], bool cpp) {
// tokenize..
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
SimpleTokenizer tokenizer(settings, *this, cpp);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// call all "runChecks" in all registered Check classes
for (auto it = Check::instances().cbegin(); it != Check::instances().cend(); ++it) {
Expand Down Expand Up @@ -396,8 +396,8 @@ class TestGarbage : public TestFixture {
const char code[] = "class x y { };";

{
SimpleTokenizer tokenizer(settings, *this);
ASSERT(tokenizer.tokenize(code, false));
SimpleTokenizer tokenizer(settings, *this, false);
ASSERT(tokenizer.tokenize(code));
ASSERT_EQUALS("", errout_str());
}
{
Expand Down
4 changes: 2 additions & 2 deletions test/testio.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,8 @@ class TestIO : public TestFixture {
settings1.platform.defaultSign = options.defaultSign;

// Tokenize..
SimpleTokenizer tokenizer(settings1, *this);
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
SimpleTokenizer tokenizer(settings1, *this, options.cpp);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
if (options.onlyFormatStr) {
Expand Down
8 changes: 4 additions & 4 deletions test/testleakautovar.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,8 +226,8 @@ class TestLeakAutoVar : public TestFixture {
const Settings settings1 = settingsBuilder(options.s ? *options.s : settings).checkLibrary().build();

// Tokenize..
SimpleTokenizer tokenizer(settings1, *this);
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
SimpleTokenizer tokenizer(settings1, *this, options.cpp);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check for leaks..
runChecks<CheckLeakAutoVar>(tokenizer, this);
Expand Down Expand Up @@ -3346,8 +3346,8 @@ class TestLeakAutoVarWindows : public TestFixture {
template<size_t size>
void check_(const char* file, int line, const char (&code)[size]) {
// Tokenize..
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code, false), file, line);
SimpleTokenizer tokenizer(settings, *this, false);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check for leaks..
runChecks<CheckLeakAutoVar>(tokenizer, this);
Expand Down
45 changes: 30 additions & 15 deletions test/testlibrary.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -872,7 +872,8 @@ class TestLibrary : public TestFixture {
ASSERT(!library.detectContainerOrIterator(nullptr));

{
const SimpleTokenizer var(*this, "std::A<int> a;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("std::A<int> a;"));
ASSERT_EQUALS(&A, library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -881,14 +882,16 @@ class TestLibrary : public TestFixture {
}

{
const SimpleTokenizer var(*this, "std::A<int>::size_type a_s;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("std::A<int>::size_type a_s;"));
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
}

{
const SimpleTokenizer var(*this, "std::A<int>::iterator a_it;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("std::A<int>::iterator a_it;"));
ASSERT(!library.detectContainer(var.tokens()));
ASSERT_EQUALS(&A, library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -897,7 +900,8 @@ class TestLibrary : public TestFixture {
}

{
const SimpleTokenizer var(*this, "std::B<int> b;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("std::B<int> b;"));
ASSERT_EQUALS(&B, library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -906,14 +910,16 @@ class TestLibrary : public TestFixture {
}

{
const SimpleTokenizer var(*this, "std::B<int>::size_type b_s;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("std::B<int>::size_type b_s;"));
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
}

{
const SimpleTokenizer var(*this, "std::B<int>::iterator b_it;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("std::B<int>::iterator b_it;"));
ASSERT(!library.detectContainer(var.tokens()));
ASSERT_EQUALS(&B, library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -922,21 +928,24 @@ class TestLibrary : public TestFixture {
}

{
const SimpleTokenizer var(*this, "C c;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("C c;"));
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
}

{
const SimpleTokenizer var(*this, "D d;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("D d;"));
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
}

{
const SimpleTokenizer var(*this, "std::E e;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("std::E e;"));
ASSERT(library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -946,31 +955,35 @@ class TestLibrary : public TestFixture {
}

{
const SimpleTokenizer var(*this, "E e;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("E e;"));
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
ASSERT_EQUALS(&E, library.detectContainerOrIterator(var.tokens(), nullptr, true));
}

{
const SimpleTokenizer var(*this, "std::E::iterator I;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("std::E::iterator I;"));
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens(), nullptr, true));
}

{
const SimpleTokenizer var(*this, "std::E::size_type p;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("std::E::size_type p;"));
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens(), nullptr, true));
}

{
const SimpleTokenizer var(*this, "std::F f;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("std::F f;"));
ASSERT(library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -979,7 +992,8 @@ class TestLibrary : public TestFixture {
}

{
const SimpleTokenizer var(*this, "std::F::iterator I;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("std::F::iterator I;"));
ASSERT(!library.detectContainer(var.tokens()));
TODO_ASSERT(library.detectIterator(var.tokens()));
bool isIterator = false;
Expand All @@ -988,7 +1002,8 @@ class TestLibrary : public TestFixture {
}

{
const SimpleTokenizer var(*this, "F::iterator I;");
SimpleTokenizer var(*this);
ASSERT(var.tokenize("F::iterator I;"));
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
Expand Down
4 changes: 2 additions & 2 deletions test/testmemleak.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1648,8 +1648,8 @@ class TestMemleakStructMember : public TestFixture {
template<size_t size>
void check_(const char* file, int line, const char (&code)[size], bool cpp = true) {
// Tokenize..
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
SimpleTokenizer tokenizer(settings, *this, cpp);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check for memory leaks..
CheckMemoryLeakStructMember checkMemoryLeakStructMember(&tokenizer, &settings, this);
Expand Down
Loading
Loading