From ba9648c23a03c392e27b0dfbd3493424afe0a040 Mon Sep 17 00:00:00 2001 From: firewave Date: Tue, 15 Apr 2025 08:23:41 +0200 Subject: [PATCH 1/4] helpers.h: removed some unused `SimpleTokenizer` code --- test/helpers.h | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/test/helpers.h b/test/helpers.h index 3e051780ff2..9dce115f822 100644 --- a/test/helpers.h +++ b/test/helpers.h @@ -56,48 +56,34 @@ class SimpleTokenizer : public Tokenizer { : Tokenizer{settings, errorlogger} {} - /* - Token* tokens() { - return Tokenizer::tokens(); - } - - const Token* tokens() const { - return Tokenizer::tokens(); - } - */ - template bool tokenize(const char (&code)[size], - const std::string& filename, - const std::string &configuration = "") + const std::string& filename) { std::istringstream istr(code); - return tokenize(istr, filename, configuration); + return tokenize(istr, filename); } template bool tokenize(const char (&code)[size], - bool cpp = true, - const std::string &configuration = "") + bool cpp = true) { std::istringstream istr(code); - return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c"), configuration); + return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c")); } bool tokenize(const std::string& code, - const std::string& filename, - const std::string &configuration = "") + const std::string& filename) { std::istringstream istr(code); - return tokenize(istr, filename, configuration); + return tokenize(istr, filename); } bool tokenize(const std::string& code, - bool cpp = true, - const std::string &configuration = "") + bool cpp = true) { std::istringstream istr(code); - return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c"), configuration); + return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c")); } private: @@ -105,12 +91,10 @@ class SimpleTokenizer : public Tokenizer { * Tokenize code * @param istr The code as stream * @param filename Indicates if the code is C++ - * @param configuration E.g. "A" for code where "#ifdef A" is true * @return false if source code contains syntax errors */ bool tokenize(std::istream& istr, - const std::string& filename, - const std::string &configuration = "") + const std::string& filename) { if (list.front()) throw std::runtime_error("token list is not empty"); @@ -118,7 +102,7 @@ class SimpleTokenizer : public Tokenizer { if (!list.createTokens(istr, Path::identify(filename, false))) return false; - return simplifyTokens1(configuration); + return simplifyTokens1(""); } // TODO: find a better solution From 8bb1416dd227ea3fa0e47a018295ddc1fe016158 Mon Sep 17 00:00:00 2001 From: firewave Date: Tue, 15 Apr 2025 08:25:18 +0200 Subject: [PATCH 2/4] testrunner: removed duplicated language parameter from `SimpleTokenizer` --- test/helpers.h | 21 ++++++++++++--------- test/test64bit.cpp | 4 ++-- test/testastutils.cpp | 4 ++-- test/testautovariables.cpp | 4 ++-- test/testbool.cpp | 4 ++-- test/testbufferoverrun.cpp | 4 ++-- test/testfunctions.cpp | 4 ++-- test/testgarbage.cpp | 8 ++++---- test/testio.cpp | 4 ++-- test/testleakautovar.cpp | 8 ++++---- test/testmemleak.cpp | 4 ++-- test/testnullpointer.cpp | 8 ++++---- test/testother.cpp | 8 ++++---- test/testsimplifytokens.cpp | 8 ++++---- test/testsymboldatabase.cpp | 20 ++++++++++---------- test/testtokenize.cpp | 34 +++++++++++++++++----------------- test/testuninitvar.cpp | 8 ++++---- test/testunusedfunctions.cpp | 4 ++-- test/testunusedvar.cpp | 8 ++++---- test/testvalueflow.cpp | 4 ++-- test/testvarid.cpp | 8 ++++---- 21 files changed, 91 insertions(+), 88 deletions(-) diff --git a/test/helpers.h b/test/helpers.h index 9dce115f822..ff13f7879a1 100644 --- a/test/helpers.h +++ b/test/helpers.h @@ -48,42 +48,45 @@ class SimpleTokenizer : public Tokenizer { SimpleTokenizer(ErrorLogger& errorlogger, const char (&code)[size], bool cpp = true) : Tokenizer{s_settings, errorlogger} { - if (!tokenize(code, cpp)) + list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true); + if (!tokenize(code)) throw std::runtime_error("creating tokens failed"); } - SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger) + SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, bool cpp = true) : Tokenizer{settings, errorlogger} - {} + { + list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true); + } template bool tokenize(const char (&code)[size], const std::string& filename) { + list.setLang(Path::identify(filename, false), true); std::istringstream istr(code); return tokenize(istr, filename); } template - bool tokenize(const char (&code)[size], - bool cpp = true) + bool tokenize(const char (&code)[size]) { std::istringstream istr(code); - return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c")); + return tokenize(istr, std::string(list.isCPP() ? "test.cpp" : "test.c")); } bool tokenize(const std::string& code, const std::string& filename) { + list.setLang(Path::identify(filename, false), true); std::istringstream istr(code); return tokenize(istr, filename); } - bool tokenize(const std::string& code, - bool cpp = true) + bool tokenize(const std::string& code) { std::istringstream istr(code); - return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c")); + return tokenize(istr, std::string(list.isCPP() ? "test.cpp" : "test.c")); } private: diff --git a/test/test64bit.cpp b/test/test64bit.cpp index 4cd67498a78..7a05bf75178 100644 --- a/test/test64bit.cpp +++ b/test/test64bit.cpp @@ -46,8 +46,8 @@ class Test64BitPortability : public TestFixture { template void check_(const char* file, int line, const char (&code)[size], bool cpp = true) { // Tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(settings, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check char variable usage.. Check64BitPortability check64BitPortability(&tokenizer, &settings, this); diff --git a/test/testastutils.cpp b/test/testastutils.cpp index e0c9a33498c..89818f72721 100644 --- a/test/testastutils.cpp +++ b/test/testastutils.cpp @@ -174,8 +174,8 @@ class TestAstUtils : public TestFixture { #define isSameExpression(...) isSameExpression_(__FILE__, __LINE__, __VA_ARGS__) template bool isSameExpression_(const char* file, int line, const char (&code)[size], const char tokStr1[], const char tokStr2[], bool cpp) { - SimpleTokenizer tokenizer(settingsDefault, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(settingsDefault, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); const Token * const tok1 = Token::findsimplematch(tokenizer.tokens(), tokStr1, strlen(tokStr1)); const Token * const tok2 = Token::findsimplematch(tok1->next(), tokStr2, strlen(tokStr2)); return (isSameExpression)(false, tok1, tok2, settingsDefault, false, true); diff --git a/test/testautovariables.cpp b/test/testautovariables.cpp index 0c37b7505e1..e44ef023387 100644 --- a/test/testautovariables.cpp +++ b/test/testautovariables.cpp @@ -44,8 +44,8 @@ class TestAutoVariables : public TestFixture { const Settings settings1 = settingsBuilder(settings).certainty(Certainty::inconclusive, options.inconclusive).build(); // Tokenize.. - SimpleTokenizer tokenizer(settings1, *this); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + SimpleTokenizer tokenizer(settings1, *this, options.cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); runChecks(tokenizer, this); } diff --git a/test/testbool.cpp b/test/testbool.cpp index 44031ca6d73..dd43cf9b19a 100644 --- a/test/testbool.cpp +++ b/test/testbool.cpp @@ -86,8 +86,8 @@ class TestBool : public TestFixture { template void check_(const char* file, int line, const char (&code)[size], const CheckOptions& options = make_default_obj()) { // Tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + SimpleTokenizer tokenizer(settings, *this, options.cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check... runChecks(tokenizer, this); diff --git a/test/testbufferoverrun.cpp b/test/testbufferoverrun.cpp index 5e97968b4f5..3ecd296508a 100644 --- a/test/testbufferoverrun.cpp +++ b/test/testbufferoverrun.cpp @@ -51,8 +51,8 @@ class TestBufferOverrun : public TestFixture { const Settings settings = options.s ? *options.s : settingsBuilder(settings0).certainty(Certainty::inconclusive).build(); // Tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + SimpleTokenizer tokenizer(settings, *this, options.cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check for buffer overruns.. runChecks(tokenizer, this); diff --git a/test/testfunctions.cpp b/test/testfunctions.cpp index 1017546d3d6..f596037c831 100644 --- a/test/testfunctions.cpp +++ b/test/testfunctions.cpp @@ -126,8 +126,8 @@ class TestFunctions : public TestFixture { const Settings& s = options.s ? *options.s : settings; // Tokenize.. - SimpleTokenizer tokenizer(s, *this); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + SimpleTokenizer tokenizer(s, *this, options.cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); runChecks(tokenizer, this); } diff --git a/test/testgarbage.cpp b/test/testgarbage.cpp index cb945f21254..321d922f2f4 100644 --- a/test/testgarbage.cpp +++ b/test/testgarbage.cpp @@ -288,8 +288,8 @@ class TestGarbage : public TestFixture { template std::string checkCodeInternal_(const char* file, int line, const char (&code)[size], bool cpp) { // tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(settings, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // call all "runChecks" in all registered Check classes for (auto it = Check::instances().cbegin(); it != Check::instances().cend(); ++it) { @@ -396,8 +396,8 @@ class TestGarbage : public TestFixture { const char code[] = "class x y { };"; { - SimpleTokenizer tokenizer(settings, *this); - ASSERT(tokenizer.tokenize(code, false)); + SimpleTokenizer tokenizer(settings, *this, false); + ASSERT(tokenizer.tokenize(code)); ASSERT_EQUALS("", errout_str()); } { diff --git a/test/testio.cpp b/test/testio.cpp index 6701b54d068..61a78430821 100644 --- a/test/testio.cpp +++ b/test/testio.cpp @@ -102,8 +102,8 @@ class TestIO : public TestFixture { settings1.platform.defaultSign = options.defaultSign; // Tokenize.. - SimpleTokenizer tokenizer(settings1, *this); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + SimpleTokenizer tokenizer(settings1, *this, options.cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check.. if (options.onlyFormatStr) { diff --git a/test/testleakautovar.cpp b/test/testleakautovar.cpp index 0ac53a8fbbd..92e28a2be1b 100644 --- a/test/testleakautovar.cpp +++ b/test/testleakautovar.cpp @@ -226,8 +226,8 @@ class TestLeakAutoVar : public TestFixture { const Settings settings1 = settingsBuilder(options.s ? *options.s : settings).checkLibrary().build(); // Tokenize.. - SimpleTokenizer tokenizer(settings1, *this); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + SimpleTokenizer tokenizer(settings1, *this, options.cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check for leaks.. runChecks(tokenizer, this); @@ -3346,8 +3346,8 @@ class TestLeakAutoVarWindows : public TestFixture { template void check_(const char* file, int line, const char (&code)[size]) { // Tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, false), file, line); + SimpleTokenizer tokenizer(settings, *this, false); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check for leaks.. runChecks(tokenizer, this); diff --git a/test/testmemleak.cpp b/test/testmemleak.cpp index abe72448f21..d6d8cc1b53c 100644 --- a/test/testmemleak.cpp +++ b/test/testmemleak.cpp @@ -1648,8 +1648,8 @@ class TestMemleakStructMember : public TestFixture { template void check_(const char* file, int line, const char (&code)[size], bool cpp = true) { // Tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(settings, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check for memory leaks.. CheckMemoryLeakStructMember checkMemoryLeakStructMember(&tokenizer, &settings, this); diff --git a/test/testnullpointer.cpp b/test/testnullpointer.cpp index 8f8e7d6d6c1..2ff38deb928 100644 --- a/test/testnullpointer.cpp +++ b/test/testnullpointer.cpp @@ -189,8 +189,8 @@ class TestNullPointer : public TestFixture { const Settings settings1 = settingsBuilder(settings).certainty(Certainty::inconclusive, options.inconclusive).build(); // Tokenize.. - SimpleTokenizer tokenizer(settings1, *this); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + SimpleTokenizer tokenizer(settings1, *this, options.cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check for null pointer dereferences.. runChecks(tokenizer, this); @@ -4212,9 +4212,9 @@ class TestNullPointer : public TestFixture { } void functioncalllibrary() { - SimpleTokenizer tokenizer(settingsDefault,*this); + SimpleTokenizer tokenizer(settingsDefault,*this,false); const char code[] = "void f() { int a,b,c; x(a,b,c); }"; - ASSERT_EQUALS(true, tokenizer.tokenize(code, false)); + ASSERT_EQUALS(true, tokenizer.tokenize(code)); const Token *xtok = Token::findsimplematch(tokenizer.tokens(), "x"); // nothing bad.. diff --git a/test/testother.cpp b/test/testother.cpp index c3b850b5e5d..41538204817 100644 --- a/test/testother.cpp +++ b/test/testother.cpp @@ -323,8 +323,8 @@ class TestOther : public TestFixture { settings->verbose = verbose; // Tokenize.. - SimpleTokenizer tokenizer(*settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(*settings, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check.. runChecks(tokenizer, this); @@ -11580,8 +11580,8 @@ class TestOther : public TestFixture { settings->verbose = verbose; // Tokenize.. - SimpleTokenizer tokenizer(*settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(*settings, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check.. runChecks(tokenizer, this); diff --git a/test/testsimplifytokens.cpp b/test/testsimplifytokens.cpp index 16b8a426395..065b8184425 100644 --- a/test/testsimplifytokens.cpp +++ b/test/testsimplifytokens.cpp @@ -174,9 +174,9 @@ class TestSimplifyTokens : public TestFixture { template std::string tok_(const char* file, int line, const char (&code)[size], const TokOptions& options = make_default_obj()) { const Settings settings = settingsBuilder(settings0).platform(options.type).build(); - SimpleTokenizer tokenizer(settings, *this); + SimpleTokenizer tokenizer(settings, *this, options.cpp); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + ASSERT_LOC(tokenizer.tokenize(code), file, line); return tokenizer.tokens()->stringifyList(nullptr, false); } @@ -194,8 +194,8 @@ class TestSimplifyTokens : public TestFixture { const Settings settings = settingsBuilder(settings1).debugwarnings().platform(options.platform).cpp(Standards::CPP03).build(); // tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, linenr); + SimpleTokenizer tokenizer(settings, *this, options.cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, linenr); if (tokenizer.tokens()) return tokenizer.tokens()->stringifyList(false, true, false, true, false, nullptr, nullptr); diff --git a/test/testsymboldatabase.cpp b/test/testsymboldatabase.cpp index 3edf11411ed..9a42d813f59 100644 --- a/test/testsymboldatabase.cpp +++ b/test/testsymboldatabase.cpp @@ -43,18 +43,18 @@ class TestSymbolDatabase; #define GET_SYMBOL_DB(code) \ SimpleTokenizer tokenizer(settings1, *this); \ - const SymbolDatabase *db = getSymbolDB_inner(tokenizer, code, true); \ + const SymbolDatabase *db = getSymbolDB_inner(tokenizer, code); \ ASSERT(db); \ do {} while (false) #define GET_SYMBOL_DB_C(code) \ - SimpleTokenizer tokenizer(settings1, *this); \ - const SymbolDatabase *db = getSymbolDB_inner(tokenizer, code, false); \ + SimpleTokenizer tokenizer(settings1, *this, false); \ + const SymbolDatabase *db = getSymbolDB_inner(tokenizer, code); \ do {} while (false) #define GET_SYMBOL_DB_DBG(code) \ SimpleTokenizer tokenizer(settingsDbg, *this); \ - const SymbolDatabase *db = getSymbolDB_inner(tokenizer, code, true); \ + const SymbolDatabase *db = getSymbolDB_inner(tokenizer, code); \ ASSERT(db); \ do {} while (false) @@ -75,8 +75,8 @@ class TestSymbolDatabase : public TestFixture { } template - static const SymbolDatabase* getSymbolDB_inner(SimpleTokenizer& tokenizer, const char (&code)[size], bool cpp) { - return tokenizer.tokenize(code, cpp) ? tokenizer.getSymbolDatabase() : nullptr; + static const SymbolDatabase* getSymbolDB_inner(SimpleTokenizer& tokenizer, const char (&code)[size]) { + return tokenizer.tokenize(code) ? tokenizer.getSymbolDatabase() : nullptr; } static const Token* findToken(Tokenizer& tokenizer, const std::string& expr, unsigned int exprline) @@ -2634,8 +2634,8 @@ class TestSymbolDatabase : public TestFixture { const Settings settings = settingsBuilder(pSettings ? *pSettings : settings1).debugwarnings(debug).build(); // Tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(settings, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // force symbol database creation tokenizer.createSymbolDatabase(); @@ -9251,8 +9251,8 @@ class TestSymbolDatabase : public TestFixture { #define typeOf(...) typeOf_(__FILE__, __LINE__, __VA_ARGS__) template std::string typeOf_(const char* file, int line, const char (&code)[size], const char pattern[], bool cpp = true, const Settings *settings = nullptr) { - SimpleTokenizer tokenizer(settings ? *settings : settings2, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(settings ? *settings : settings2, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); const Token* tok; for (tok = tokenizer.list.back(); tok; tok = tok->previous()) if (Token::simpleMatch(tok, pattern, strlen(pattern))) diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index a97e1981b95..9c638348c4c 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -489,8 +489,8 @@ class TestTokenizer : public TestFixture { const Settings settings = settingsBuilder(settings1).debugwarnings().cpp(cppstd).c(cstd).platform(platform).build(); // tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, linenr); + SimpleTokenizer tokenizer(settings, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, linenr); if (tokenizer.tokens()) return tokenizer.tokens()->stringifyList(false, expand, false, true, false, nullptr, nullptr); @@ -516,8 +516,8 @@ class TestTokenizer : public TestFixture { const Settings settings = settingsBuilder(settings_windows).debugwarnings().cpp(cpp11 ? Standards::CPP11 : Standards::CPP03).platform(platform).build(); // tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, linenr); + SimpleTokenizer tokenizer(settings, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, linenr); if (tokenizer.tokens()) return tokenizer.tokens()->stringifyList(false, expand, false, true, false, nullptr, nullptr); @@ -527,8 +527,8 @@ class TestTokenizer : public TestFixture { template std::string tokenizeAndStringify_(const char* file, int line, const char (&code)[size], const Settings &settings, bool cpp = true) { // tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(settings, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); if (!tokenizer.tokens()) return ""; return tokenizer.tokens()->stringifyList(false, true, false, true, false, nullptr, nullptr); @@ -539,8 +539,8 @@ class TestTokenizer : public TestFixture { std::string tokenizeDebugListing_(const char* file, int line, const char (&code)[size], bool cpp = true) { const Settings settings = settingsBuilder(settings0).c(Standards::C89).cpp(Standards::CPP03).build(); - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(settings, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // result.. return tokenizer.tokens()->stringifyList(true,true,true,true,false); @@ -8026,8 +8026,8 @@ class TestTokenizer : public TestFixture { void dumpAlignas() { Settings settings; - SimpleTokenizer tokenizer(settings, *this); - ASSERT(tokenizer.tokenize("int alignas(8) alignas(16) x;", false)); + SimpleTokenizer tokenizer(settings, *this, false); + ASSERT(tokenizer.tokenize("int alignas(8) alignas(16) x;")); ASSERT(Token::simpleMatch(tokenizer.tokens(), "int x ;")); std::ostringstream ostr; tokenizer.dump(ostr); @@ -8425,8 +8425,8 @@ class TestTokenizer : public TestFixture { const char code[] = "void f(void) {\n" " double result = (strtod)(\"NAN\", NULL);\n" "}\n"; - SimpleTokenizer tokenizer(settings1, *this); - ASSERT_LOC(tokenizer.tokenize(code, false), __FILE__, __LINE__); + SimpleTokenizer tokenizer(settings1, *this, false); + ASSERT_LOC(tokenizer.tokenize(code), __FILE__, __LINE__); const Token *f = Token::findsimplematch(tokenizer.tokens(), "strtod"); ASSERT(f); ASSERT(!f->previous()->isCast()); @@ -8437,7 +8437,7 @@ class TestTokenizer : public TestFixture { " return static_cast(std::ceil((std::min)(a, (std::min)(b, c))));\n" "}\n"; SimpleTokenizer tokenizer(settings1, *this); - ASSERT_LOC(tokenizer.tokenize(code, true), __FILE__, __LINE__); + ASSERT_LOC(tokenizer.tokenize(code), __FILE__, __LINE__); const Token *f = Token::findsimplematch(tokenizer.tokens(), "min"); ASSERT(f); const Token *par = f->next(); @@ -8452,7 +8452,7 @@ class TestTokenizer : public TestFixture { " return (a != 0) ? 1 : (std::min)(1, b);\n" "}\n"; SimpleTokenizer tokenizer(settings1, *this); - ASSERT_LOC(tokenizer.tokenize(code, true), __FILE__, __LINE__); + ASSERT_LOC(tokenizer.tokenize(code), __FILE__, __LINE__); const Token *f = Token::findsimplematch(tokenizer.tokens(), "min"); ASSERT(f); const Token *par = f->next(); @@ -8523,8 +8523,6 @@ class TestTokenizer : public TestFixture { } void dumpFallthrough() { - Settings settings; - SimpleTokenizer tokenizer(settings, *this); const char * code = "void f(int n) {\n" " void g(), h(), i();\n" " switch (n) {\n" @@ -8540,7 +8538,9 @@ class TestTokenizer : public TestFixture { " break;\n" " }\n" "}"; - ASSERT(tokenizer.tokenize(code, false)); + Settings settings; + SimpleTokenizer tokenizer(settings, *this, false); + ASSERT(tokenizer.tokenize(code)); std::ostringstream ostr; tokenizer.dump(ostr); const std::string dump = ostr.str(); diff --git a/test/testuninitvar.cpp b/test/testuninitvar.cpp index aa17e99247a..863518b6ce2 100644 --- a/test/testuninitvar.cpp +++ b/test/testuninitvar.cpp @@ -118,8 +118,8 @@ class TestUninitVar : public TestFixture { const Settings settings1 = settingsBuilder(options.s ? *options.s : settings).debugwarnings(options.debugwarnings).build(); // Tokenize.. - SimpleTokenizer tokenizer(settings1, *this); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + SimpleTokenizer tokenizer(settings1, *this, options.cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check for redundant code.. CheckUninitVar checkuninitvar(&tokenizer, &settings1, this); @@ -5464,8 +5464,8 @@ class TestUninitVar : public TestFixture { // Tokenize.. const Settings s = settingsBuilder(settings).debugwarnings(false).build(); - SimpleTokenizer tokenizer(s, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(s, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check for redundant code.. CheckUninitVar checkuninitvar(&tokenizer, &s, this); diff --git a/test/testunusedfunctions.cpp b/test/testunusedfunctions.cpp index e564eb71f43..a61ef62b8a6 100644 --- a/test/testunusedfunctions.cpp +++ b/test/testunusedfunctions.cpp @@ -103,8 +103,8 @@ class TestUnusedFunctions : public TestFixture { const Settings settings1 = settingsBuilder(options.s ? *options.s : settings).platform(options.platform).build(); // Tokenize.. - SimpleTokenizer tokenizer(settings1, *this); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + SimpleTokenizer tokenizer(settings1, *this, options.cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check for unused functions.. CheckUnusedFunctions checkUnusedFunctions; diff --git a/test/testunusedvar.cpp b/test/testunusedvar.cpp index 4a13479bf09..e7779c69d1b 100644 --- a/test/testunusedvar.cpp +++ b/test/testunusedvar.cpp @@ -267,8 +267,8 @@ class TestUnusedVar : public TestFixture { #define functionVariableUsage(...) functionVariableUsage_(__FILE__, __LINE__, __VA_ARGS__) void functionVariableUsage_(const char* file, int line, const char code[], const FunctionVariableUsageOptions& options = make_default_obj()) { // Tokenize.. - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + SimpleTokenizer tokenizer(settings, *this, options.cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check for unused variables.. CheckUnusedVar checkUnusedVar(&tokenizer, &settings, this); @@ -285,10 +285,10 @@ class TestUnusedVar : public TestFixture { #define checkStructMemberUsage(...) checkStructMemberUsage_(__FILE__, __LINE__, __VA_ARGS__) void checkStructMemberUsage_(const char* file, int line, const char code[], const CheckStructMemberUsageOptions& options = make_default_obj()) { // Tokenize.. - SimpleTokenizer tokenizer(settings, *this); + SimpleTokenizer tokenizer(settings, *this, options.cpp); if (options.directives) tokenizer.setDirectives(*options.directives); - ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line); + ASSERT_LOC(tokenizer.tokenize(code), file, line); // Check for unused variables.. CheckUnusedVar checkUnusedVar(&tokenizer, &settings, this); diff --git a/test/testvalueflow.cpp b/test/testvalueflow.cpp index 29bf5091581..b1435845477 100644 --- a/test/testvalueflow.cpp +++ b/test/testvalueflow.cpp @@ -491,8 +491,8 @@ class TestValueFlow : public TestFixture { #define tokenValues(...) tokenValues_(__FILE__, __LINE__, __VA_ARGS__) std::list tokenValues_(const char* file, int line, const char code[], const char tokstr[], const Settings *s = nullptr, bool cpp = true) { - SimpleTokenizer tokenizer(s ? *s : settings, *this); - ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line); + SimpleTokenizer tokenizer(s ? *s : settings, *this, cpp); + ASSERT_LOC(tokenizer.tokenize(code), file, line); const Token *tok = Token::findmatch(tokenizer.tokens(), tokstr); return tok ? tok->values() : std::list(); } diff --git a/test/testvarid.cpp b/test/testvarid.cpp index c6dbc626c8c..4db197ab9e6 100644 --- a/test/testvarid.cpp +++ b/test/testvarid.cpp @@ -271,8 +271,8 @@ class TestVarID : public TestFixture { std::string tokenize_(const char* file, int line, const char (&code)[size], const TokenizeOptions& options = make_default_obj()) { const Settings *settings1 = options.s ? options.s : &settings; - SimpleTokenizer tokenizer(*settings1, *this); - ASSERT_LOC((tokenizer.tokenize)(code, options.cpp), file, line); + SimpleTokenizer tokenizer(*settings1, *this, options.cpp); + ASSERT_LOC((tokenizer.tokenize)(code), file, line); // result.. Token::stringifyOptions str_options = Token::stringifyOptions::forDebugVarId(); @@ -308,8 +308,8 @@ class TestVarID : public TestFixture { #define compareVaridsForVariable(...) compareVaridsForVariable_(__FILE__, __LINE__, __VA_ARGS__) template std::string compareVaridsForVariable_(const char* file, int line, const char (&code)[size], const char varname[], bool cpp = true) { - SimpleTokenizer tokenizer(settings, *this); - ASSERT_LOC((tokenizer.tokenize)(code, cpp), file, line); + SimpleTokenizer tokenizer(settings, *this, cpp); + ASSERT_LOC((tokenizer.tokenize)(code), file, line); unsigned int varid = ~0U; for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next()) { From b48d86d6cde326640fefc5ba3fbbc852ce029728 Mon Sep 17 00:00:00 2001 From: firewave Date: Tue, 15 Apr 2025 08:46:11 +0200 Subject: [PATCH 3/4] testrunner: moved `tokenize()` call out of a `SimpleTokenizer` constructor --- test/helpers.h | 5 +-- test/testlibrary.cpp | 45 +++++++++++++++-------- test/testsymboldatabase.cpp | 3 +- test/testtoken.cpp | 72 +++++++++++++++++++++++-------------- 4 files changed, 79 insertions(+), 46 deletions(-) diff --git a/test/helpers.h b/test/helpers.h index ff13f7879a1..cf2bfe94890 100644 --- a/test/helpers.h +++ b/test/helpers.h @@ -44,13 +44,10 @@ namespace tinyxml2 { // TODO: make Tokenizer private class SimpleTokenizer : public Tokenizer { public: - template - SimpleTokenizer(ErrorLogger& errorlogger, const char (&code)[size], bool cpp = true) + explicit SimpleTokenizer(ErrorLogger& errorlogger, bool cpp = true) : Tokenizer{s_settings, errorlogger} { list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true); - if (!tokenize(code)) - throw std::runtime_error("creating tokens failed"); } SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, bool cpp = true) diff --git a/test/testlibrary.cpp b/test/testlibrary.cpp index 002237a39ba..b1b94c338e1 100644 --- a/test/testlibrary.cpp +++ b/test/testlibrary.cpp @@ -872,7 +872,8 @@ class TestLibrary : public TestFixture { ASSERT(!library.detectContainerOrIterator(nullptr)); { - const SimpleTokenizer var(*this, "std::A a;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::A a;")); ASSERT_EQUALS(&A, library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); bool isIterator; @@ -881,14 +882,16 @@ class TestLibrary : public TestFixture { } { - const SimpleTokenizer var(*this, "std::A::size_type a_s;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::A::size_type a_s;")); ASSERT(!library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); ASSERT(!library.detectContainerOrIterator(var.tokens())); } { - const SimpleTokenizer var(*this, "std::A::iterator a_it;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::A::iterator a_it;")); ASSERT(!library.detectContainer(var.tokens())); ASSERT_EQUALS(&A, library.detectIterator(var.tokens())); bool isIterator; @@ -897,7 +900,8 @@ class TestLibrary : public TestFixture { } { - const SimpleTokenizer var(*this, "std::B b;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::B b;")); ASSERT_EQUALS(&B, library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); bool isIterator; @@ -906,14 +910,16 @@ class TestLibrary : public TestFixture { } { - const SimpleTokenizer var(*this, "std::B::size_type b_s;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::B::size_type b_s;")); ASSERT(!library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); ASSERT(!library.detectContainerOrIterator(var.tokens())); } { - const SimpleTokenizer var(*this, "std::B::iterator b_it;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::B::iterator b_it;")); ASSERT(!library.detectContainer(var.tokens())); ASSERT_EQUALS(&B, library.detectIterator(var.tokens())); bool isIterator; @@ -922,21 +928,24 @@ class TestLibrary : public TestFixture { } { - const SimpleTokenizer var(*this, "C c;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("C c;")); ASSERT(!library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); ASSERT(!library.detectContainerOrIterator(var.tokens())); } { - const SimpleTokenizer var(*this, "D d;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("D d;")); ASSERT(!library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); ASSERT(!library.detectContainerOrIterator(var.tokens())); } { - const SimpleTokenizer var(*this, "std::E e;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::E e;")); ASSERT(library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); bool isIterator; @@ -946,7 +955,8 @@ class TestLibrary : public TestFixture { } { - const SimpleTokenizer var(*this, "E e;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("E e;")); ASSERT(!library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); ASSERT(!library.detectContainerOrIterator(var.tokens())); @@ -954,7 +964,8 @@ class TestLibrary : public TestFixture { } { - const SimpleTokenizer var(*this, "std::E::iterator I;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::E::iterator I;")); ASSERT(!library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); ASSERT(!library.detectContainerOrIterator(var.tokens())); @@ -962,7 +973,8 @@ class TestLibrary : public TestFixture { } { - const SimpleTokenizer var(*this, "std::E::size_type p;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::E::size_type p;")); ASSERT(!library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); ASSERT(!library.detectContainerOrIterator(var.tokens())); @@ -970,7 +982,8 @@ class TestLibrary : public TestFixture { } { - const SimpleTokenizer var(*this, "std::F f;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::F f;")); ASSERT(library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); bool isIterator; @@ -979,7 +992,8 @@ class TestLibrary : public TestFixture { } { - const SimpleTokenizer var(*this, "std::F::iterator I;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::F::iterator I;")); ASSERT(!library.detectContainer(var.tokens())); TODO_ASSERT(library.detectIterator(var.tokens())); bool isIterator = false; @@ -988,7 +1002,8 @@ class TestLibrary : public TestFixture { } { - const SimpleTokenizer var(*this, "F::iterator I;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("F::iterator I;")); ASSERT(!library.detectContainer(var.tokens())); ASSERT(!library.detectIterator(var.tokens())); ASSERT(!library.detectContainerOrIterator(var.tokens())); diff --git a/test/testsymboldatabase.cpp b/test/testsymboldatabase.cpp index 9a42d813f59..c79417f6cf6 100644 --- a/test/testsymboldatabase.cpp +++ b/test/testsymboldatabase.cpp @@ -846,7 +846,8 @@ class TestSymbolDatabase : public TestFixture { } { reset(); - const SimpleTokenizer constpointer(*this, "const int* p;"); + SimpleTokenizer constpointer(*this); + ASSERT(constpointer.tokenize("const int* p;")); Variable v2(constpointer.tokens()->tokAt(3), constpointer.tokens()->next(), constpointer.tokens()->tokAt(2), 0, AccessControl::Public, nullptr, nullptr, settings1); ASSERT(false == v2.isArray()); ASSERT(true == v2.isPointer()); diff --git a/test/testtoken.cpp b/test/testtoken.cpp index e627ae564e4..41fe456149f 100644 --- a/test/testtoken.cpp +++ b/test/testtoken.cpp @@ -301,7 +301,8 @@ class TestToken : public TestFixture { } void multiCompare4() { - const SimpleTokenizer var(*this, "std :: queue < int > foo ;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std :: queue < int > foo ;")); ASSERT_EQUALS(Token::eBracket, var.tokens()->tokAt(3)->tokType()); ASSERT_EQUALS(Token::eBracket, var.tokens()->tokAt(5)->tokType()); @@ -581,17 +582,21 @@ class TestToken : public TestFixture { } void nextArgument() { - const SimpleTokenizer example1(*this, "foo(1, 2, 3, 4);"); + SimpleTokenizer example1(*this); + ASSERT(example1.tokenize("foo(1, 2, 3, 4);")); ASSERT_EQUALS(true, Token::simpleMatch(example1.tokens()->tokAt(2)->nextArgument(), "2 , 3")); ASSERT_EQUALS(true, Token::simpleMatch(example1.tokens()->tokAt(4)->nextArgument(), "3 , 4")); - const SimpleTokenizer example2(*this, "foo();"); + SimpleTokenizer example2(*this); + ASSERT(example2.tokenize("foo();")); ASSERT_EQUALS(true, example2.tokens()->tokAt(2)->nextArgument() == nullptr); - const SimpleTokenizer example3(*this, "foo(bar(a, b), 2, 3);"); + SimpleTokenizer example3(*this); + ASSERT(example3.tokenize("foo(bar(a, b), 2, 3);")); ASSERT_EQUALS(true, Token::simpleMatch(example3.tokens()->tokAt(2)->nextArgument(), "2 , 3")); - const SimpleTokenizer example4(*this, "foo(x.i[1], \"\", 3);"); + SimpleTokenizer example4(*this); + ASSERT(example4.tokenize("foo(x.i[1], \"\", 3);")); ASSERT_EQUALS(true, Token::simpleMatch(example4.tokens()->tokAt(2)->nextArgument(), "\"\" , 3")); } @@ -643,7 +648,8 @@ class TestToken : public TestFixture { const SimpleTokenList type("abc"); ASSERT_EQUALS(true, Token::Match(type.front(), "%type%")); - const SimpleTokenizer isVar(*this, "int a = 3 ;"); + SimpleTokenizer isVar(*this); + ASSERT(isVar.tokenize("int a = 3 ;")); ASSERT_EQUALS(true, Token::Match(isVar.tokens(), "%type%")); ASSERT_EQUALS(true, Token::Match(isVar.tokens(), "%type% %name%")); ASSERT_EQUALS(false, Token::Match(isVar.tokens(), "%type% %type%")); @@ -697,7 +703,8 @@ class TestToken : public TestFixture { } void matchVarid() { - const SimpleTokenizer var(*this, "int a ; int b ;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("int a ; int b ;")); // Varid == 0 should throw exception ASSERT_THROW_INTERNAL_EQUALS((void)Token::Match(var.tokens(), "%type% %varid% ; %type% %name%", 0),INTERNAL,"Internal error. Token::Match called with varid 0. Please report this to Cppcheck developers"); @@ -1128,14 +1135,16 @@ class TestToken : public TestFixture { } void canFindMatchingBracketsNeedsOpen() { - const SimpleTokenizer var(*this, "std::deque > intsets;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::deque > intsets;")); const Token* const t = var.tokens()->findClosingBracket(); ASSERT(t == nullptr); } void canFindMatchingBracketsInnerPair() { - const SimpleTokenizer var(*this, "std::deque > intsets;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::deque > intsets;")); const Token * const t = var.tokens()->tokAt(7)->findClosingBracket(); ASSERT_EQUALS(">", t->str()); @@ -1143,7 +1152,8 @@ class TestToken : public TestFixture { } void canFindMatchingBracketsOuterPair() { - const SimpleTokenizer var(*this, "std::deque > intsets;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("std::deque > intsets;")); const Token* const t = var.tokens()->tokAt(3)->findClosingBracket(); ASSERT_EQUALS(">", t->str()); @@ -1151,7 +1161,8 @@ class TestToken : public TestFixture { } void canFindMatchingBracketsWithTooManyClosing() { - const SimpleTokenizer var(*this, "X< 1>2 > x1;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("X< 1>2 > x1;")); const Token* const t = var.tokens()->next()->findClosingBracket(); ASSERT_EQUALS(">", t->str()); @@ -1159,7 +1170,8 @@ class TestToken : public TestFixture { } void canFindMatchingBracketsWithTooManyOpening() { - const SimpleTokenizer var(*this, "X < (2 < 1) > x1;"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("X < (2 < 1) > x1;")); const Token* t = var.tokens()->next()->findClosingBracket(); ASSERT(t != nullptr && t->str() == ">"); @@ -1169,31 +1181,33 @@ class TestToken : public TestFixture { } void findClosingBracket() { - const SimpleTokenizer var(*this, "template struct S : public Fred> {}"); + SimpleTokenizer var(*this); + ASSERT(var.tokenize("template struct S : public Fred> {}")); const Token* const t = var.tokens()->next()->findClosingBracket(); ASSERT(Token::simpleMatch(t, "> struct")); } void findClosingBracket2() { - const SimpleTokenizer var(*this, "const auto g = []() {};\n"); // #11275 + SimpleTokenizer var(*this); // #11275 + ASSERT(var.tokenize("const auto g = []() {};\n")); const Token* const t = Token::findsimplematch(var.tokens(), "<"); ASSERT(t && Token::simpleMatch(t->findClosingBracket(), ">")); } void findClosingBracket3() { - const SimpleTokenizer var(*this, // #12789 - "template * = nullptr>\n" - "void f();\n"); + SimpleTokenizer var(*this); // #12789 + ASSERT(var.tokenize("template * = nullptr>\n" + "void f();\n")); const Token* const t = Token::findsimplematch(var.tokens(), "<"); ASSERT(t && Token::simpleMatch(t->findClosingBracket(), ">")); } void findClosingBracket4() { - const SimpleTokenizer var(*this, // #12923 - "template class T = std::vector, class U = std::vector, class V = void>\n" - "class C;\n"); + SimpleTokenizer var(*this); // #12923 + ASSERT(var.tokenize("template class T = std::vector, class U = std::vector, class V = void>\n" + "class C;\n")); const Token *const t = Token::findsimplematch(var.tokens(), "<"); ASSERT(t); const Token *const closing = t->findClosingBracket(); @@ -1201,24 +1215,30 @@ class TestToken : public TestFixture { } void expressionString() { - const SimpleTokenizer var1(*this, "void f() { *((unsigned long long *)x) = 0; }"); + SimpleTokenizer var1(*this); + ASSERT(var1.tokenize("void f() { *((unsigned long long *)x) = 0; }")); const Token *const tok1 = Token::findsimplematch(var1.tokens(), "*"); ASSERT_EQUALS("*((unsigned long long*)x)", tok1->expressionString()); - const SimpleTokenizer var2(*this, "typedef unsigned long long u64; void f() { *((u64 *)x) = 0; }"); + SimpleTokenizer var2(*this); + ASSERT(var2.tokenize("typedef unsigned long long u64; void f() { *((u64 *)x) = 0; }")); const Token *const tok2 = Token::findsimplematch(var2.tokens(), "*"); ASSERT_EQUALS("*((unsigned long long*)x)", tok2->expressionString()); - const SimpleTokenizer data3(*this, "void f() { return (t){1,2}; }"); + SimpleTokenizer data3(*this); + ASSERT(data3.tokenize("void f() { return (t){1,2}; }")); ASSERT_EQUALS("return(t){1,2}", data3.tokens()->tokAt(5)->expressionString()); - const SimpleTokenizer data4(*this, "void f() { return L\"a\"; }"); + SimpleTokenizer data4(*this); + ASSERT(data4.tokenize("void f() { return L\"a\"; }")); ASSERT_EQUALS("returnL\"a\"", data4.tokens()->tokAt(5)->expressionString()); - const SimpleTokenizer data5(*this, "void f() { return U\"a\"; }"); + SimpleTokenizer data5(*this); + ASSERT(data5.tokenize("void f() { return U\"a\"; }")); ASSERT_EQUALS("returnU\"a\"", data5.tokens()->tokAt(5)->expressionString()); - const SimpleTokenizer data6(*this, "x = \"\\0\\x1\\x2\\x3\\x4\\x5\\x6\\x7\";"); + SimpleTokenizer data6(*this); + ASSERT(data6.tokenize("x = \"\\0\\x1\\x2\\x3\\x4\\x5\\x6\\x7\";")); ASSERT_EQUALS("x=\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\"", data6.tokens()->next()->expressionString()); } From 612fbad6547165939419ae64f25cb24471727965 Mon Sep 17 00:00:00 2001 From: firewave Date: Tue, 15 Apr 2025 08:58:05 +0200 Subject: [PATCH 4/4] testrunner: got rid of `SimpleTokenizer::tokenize()` overload with filename --- test/helpers.h | 18 ++++-------------- test/testclass.cpp | 4 ++-- test/testunusedfunctions.cpp | 4 ++-- test/testvarid.cpp | 4 ++-- 4 files changed, 10 insertions(+), 20 deletions(-) diff --git a/test/helpers.h b/test/helpers.h index cf2bfe94890..b27b1e1e0c0 100644 --- a/test/helpers.h +++ b/test/helpers.h @@ -56,13 +56,11 @@ class SimpleTokenizer : public Tokenizer { list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true); } - template - bool tokenize(const char (&code)[size], - const std::string& filename) + SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const std::string& filename) + : Tokenizer{settings, errorlogger} { - list.setLang(Path::identify(filename, false), true); - std::istringstream istr(code); - return tokenize(istr, filename); + list.setLang(Path::identify(filename, false)); + list.appendFileIfNew(filename); } template @@ -72,14 +70,6 @@ class SimpleTokenizer : public Tokenizer { return tokenize(istr, std::string(list.isCPP() ? "test.cpp" : "test.c")); } - bool tokenize(const std::string& code, - const std::string& filename) - { - list.setLang(Path::identify(filename, false), true); - std::istringstream istr(code); - return tokenize(istr, filename); - } - bool tokenize(const std::string& code) { std::istringstream istr(code); diff --git a/test/testclass.cpp b/test/testclass.cpp index 11b71b14b8c..4c8286fe19f 100644 --- a/test/testclass.cpp +++ b/test/testclass.cpp @@ -9081,9 +9081,9 @@ class TestClass : public TestFixture { // getFileInfo std::list fileInfo; for (const std::string& c: code) { - SimpleTokenizer tokenizer{settingsDefault, *this}; const std::string filename = std::to_string(fileInfo.size()) + ".cpp"; - ASSERT(tokenizer.tokenize(c, filename)); + SimpleTokenizer tokenizer{settingsDefault, *this, filename}; + ASSERT(tokenizer.tokenize(c)); fileInfo.push_back(check.getFileInfo(tokenizer, settingsDefault)); } diff --git a/test/testunusedfunctions.cpp b/test/testunusedfunctions.cpp index a61ef62b8a6..b705ed91f61 100644 --- a/test/testunusedfunctions.cpp +++ b/test/testunusedfunctions.cpp @@ -596,8 +596,8 @@ class TestUnusedFunctions : public TestFixture { for (int i = 1; i <= 2; ++i) { const std::string fname = "test" + std::to_string(i) + ".cpp"; - SimpleTokenizer tokenizer{settings, *this}; - ASSERT(tokenizer.tokenize(code, fname)); + SimpleTokenizer tokenizer{settings, *this, fname}; + ASSERT(tokenizer.tokenize(code)); c.parseTokens(tokenizer, settings); } diff --git a/test/testvarid.cpp b/test/testvarid.cpp index 4db197ab9e6..b30b56c2bfb 100644 --- a/test/testvarid.cpp +++ b/test/testvarid.cpp @@ -283,8 +283,8 @@ class TestVarID : public TestFixture { #define tokenizeHeader(...) tokenizeHeader_(__FILE__, __LINE__, __VA_ARGS__) template std::string tokenizeHeader_(const char* file, int line, const char (&code)[size], const char filename[]) { - SimpleTokenizer tokenizer{settings, *this}; - ASSERT_LOC((tokenizer.tokenize)(code, std::string(filename)), file, line); + SimpleTokenizer tokenizer{settings, *this, std::string(filename)}; + ASSERT_LOC((tokenizer.tokenize)(code), file, line); // result.. Token::stringifyOptions options = Token::stringifyOptions::forDebugVarId();