Skip to content

testrunner: some Tokenizer related refactoring #7427

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
firewave merged 10 commits into from
Apr 7, 2025
Merged

testrunner: some Tokenizer related refactoring #7427

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
be3d4eb
testrunner: added `SimpleTokenizer2` wrapper
firewave Apr 3, 2025
cb1a16a
moved some code from `PreprocessorHelper` to `TestPreprocessor`
firewave Apr 3, 2025
6f29f5c
small `TestPreprocessor` cleanup
firewave Apr 3, 2025
03f6a6e
testrunner: more `SimpleTokenizer` usage
firewave Apr 3, 2025
f77f5cd
moved `preprocess()` from `PreprocessorHelper` to `SimpleTokenizer2`
firewave Apr 3, 2025
8b2c1be
testrunner: moved `getRemarkComments()` from `PreprocessorHelper` to …
firewave Apr 3, 2025
42b909f
testpreprocessor.cpp: fixed `functionConst` selfcheck warning
firewave Apr 3, 2025
cc71656
testrunner: do not pass file0 via filelist into `SimpleTokenizer2`
firewave Apr 4, 2025
9d0833b
testrunner: folded filelist object into `SimpleTokenizer2`
firewave Apr 4, 2025
8c4074f
helpers.cpp: removed obsolete TODO
firewave Apr 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 50 additions & 50 deletions Makefile
View file
Open in desktop

Large diffs are not rendered by default.

40 changes: 2 additions & 38 deletions test/helpers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,13 +162,11 @@ std::map<std::string, std::string> PreprocessorHelper::getcode(const Settings& s
return cfgcode;
}

void PreprocessorHelper::preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, ErrorLogger& errorlogger)
void SimpleTokenizer2::preprocess(const char code[], std::vector<std::string> &files, const std::string& file0, Tokenizer& tokenizer, ErrorLogger& errorlogger)
{
// TODO: make sure the given Tokenizer has not been used yet

// TODO: get rid of stream
std::istringstream istr(code);
const simplecpp::TokenList tokens1(istr, files, files[0]);
const simplecpp::TokenList tokens1(istr, files, file0);

Preprocessor preprocessor(tokenizer.getSettings(), errorlogger);
simplecpp::TokenList tokens2 = preprocessor.preprocess(tokens1, "", files, true);
Expand All @@ -180,40 +178,6 @@ void PreprocessorHelper::preprocess(const char code[], std::vector<std::string>
tokenizer.setDirectives(std::move(directives));
}

// TODO: get rid of this
void PreprocessorHelper::preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, ErrorLogger& errorlogger, const simplecpp::DUI& dui)
{
// TODO: make sure the given Tokenizer has not been used yet

std::istringstream istr(code);
const simplecpp::TokenList tokens1(istr, files, files[0]);

// Preprocess..
simplecpp::TokenList tokens2(files);
std::map<std::string, simplecpp::TokenList*> filedata;
// TODO: provide and handle outputList
simplecpp::preprocess(tokens2, tokens1, files, filedata, dui);

// Tokenizer..
tokenizer.list.createTokens(std::move(tokens2));

const Preprocessor preprocessor(tokenizer.getSettings(), errorlogger);
std::list<Directive> directives = preprocessor.createDirectives(tokens1);
tokenizer.setDirectives(std::move(directives));
}

std::vector<RemarkComment> PreprocessorHelper::getRemarkComments(const char code[], ErrorLogger& errorLogger)
{
std::vector<std::string> files{"test.cpp"};
std::istringstream istr(code);
const simplecpp::TokenList tokens1(istr, files, files[0]);

const Settings settings;

const Preprocessor preprocessor(settings, errorLogger);
return preprocessor.getRemarkComments(tokens1);
}

bool LibraryHelper::loadxmldata(Library &lib, const char xmldata[], std::size_t len)
{
tinyxml2::XMLDocument doc;
Expand Down
80 changes: 56 additions & 24 deletions test/helpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
#define helpersH

#include "library.h"
#include "preprocessor.h"
#include "settings.h"
#include "standards.h"
#include "tokenize.h"
Expand All @@ -37,9 +36,6 @@
class Token;
class SuppressionList;
class ErrorLogger;
namespace simplecpp {
struct DUI;
}
namespace tinyxml2 {
class XMLDocument;
}
Expand Down Expand Up @@ -69,39 +65,59 @@ class SimpleTokenizer : public Tokenizer {
}
*/

/**
* Tokenize code
* @param code The code
* @param cpp Indicates if the code is C++
* @param configuration E.g. "A" for code where "#ifdef A" is true
* @return false if source code contains syntax errors
*/
template<size_t size>
bool tokenize(const char (&code)[size],
Copy link
Copy Markdown
Collaborator

@danmar danmar Apr 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is this a template. I fear that for almost each test code example there will be a separate function instantiation. And it just seems redundant.

Copy link
Copy Markdown
Collaborator Author

@firewave firewave Apr 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See #7427 (comment).

Copy link
Copy Markdown
Collaborator

@danmar danmar Apr 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok I hope we can fix that later then!

const std::string& filename,
const std::string &configuration = "")
{
std::istringstream istr(code);
return tokenize(istr, filename, configuration);
}

template<size_t size>
bool tokenize(const char (&code)[size],
bool cpp = true,
const std::string &configuration = "")
{
std::istringstream istr(code);
if (!list.createTokens(istr, cpp ? "test.cpp" : "test.c"))
return false;
return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c"), configuration);
}

return simplifyTokens1(configuration);
bool tokenize(const std::string& code,
const std::string& filename,
const std::string &configuration = "")
{
std::istringstream istr(code);
return tokenize(istr, filename, configuration);
}

// TODO: get rid of this
bool tokenize(const std::string& code,
bool cpp = true,
const std::string &configuration = "")
{
std::istringstream istr(code);
if (!list.createTokens(istr, cpp ? "test.cpp" : "test.c"))
return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c"), configuration);
}

private:
/**
* Tokenize code
* @param istr The code as stream
* @param filename Indicates if the code is C++
* @param configuration E.g. "A" for code where "#ifdef A" is true
* @return false if source code contains syntax errors
*/
bool tokenize(std::istream& istr,
const std::string& filename,
const std::string &configuration = "")
{
if (!list.createTokens(istr, filename))
return false;

return simplifyTokens1(configuration);
}

private:
// TODO. find a better solution
// TODO: find a better solution
static const Settings s_settings;
};

Expand Down Expand Up @@ -171,12 +187,6 @@ class PreprocessorHelper
static std::string getcode(const Settings& settings, ErrorLogger& errorlogger, const std::string &filedata, const std::string &cfg, const std::string &filename, SuppressionList *inlineSuppression = nullptr);
static std::map<std::string, std::string> getcode(const Settings& settings, ErrorLogger& errorlogger, const char code[], const std::string &filename = "file.c", SuppressionList *inlineSuppression = nullptr);

static void preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, ErrorLogger& errorlogger);
static void preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, ErrorLogger& errorlogger, const simplecpp::DUI& dui);

/** get remark comments */
static std::vector<RemarkComment> getRemarkComments(const char code[], ErrorLogger& errorLogger);

private:
static std::map<std::string, std::string> getcode(const Settings& settings, ErrorLogger& errorlogger, const char code[], std::set<std::string> cfgs, const std::string &filename = "file.c", SuppressionList *inlineSuppression = nullptr);
};
Expand Down Expand Up @@ -246,4 +256,26 @@ struct LibraryHelper
static Library::Error loadxmldoc(Library &lib, const tinyxml2::XMLDocument& doc);
};

class SimpleTokenizer2 : public Tokenizer {
public:
template<size_t size>
SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char (&code)[size], const std::string& file0)
: Tokenizer{settings, errorlogger}
{
preprocess(code, mFiles, file0, *this, errorlogger);
}

// TODO: get rid of this
SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char code[], const std::string& file0)
: Tokenizer{settings, errorlogger}
{
preprocess(code, mFiles, file0, *this, errorlogger);
}

private:
static void preprocess(const char code[], std::vector<std::string> &files, const std::string& file0, Tokenizer& tokenizer, ErrorLogger& errorlogger);

std::vector<std::string> mFiles;
};

#endif // helpersH
5 changes: 1 addition & 4 deletions test/testbufferoverrun.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
#include "platform.h"
#include "settings.h"
#include "fixture.h"
#include "tokenize.h"

#include <cstddef>
#include <list>
Expand Down Expand Up @@ -76,9 +75,7 @@ class TestBufferOverrun : public TestFixture {
{
const Settings settings = settingsBuilder(settings0).severity(Severity::performance).certainty(Certainty::inconclusive).build();

std::vector<std::string> files(1, "test.cpp");
Tokenizer tokenizer(settings, *this);
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
SimpleTokenizer2 tokenizer(settings, *this, code, "test.cpp");

// Tokenizer..
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
Expand Down
17 changes: 4 additions & 13 deletions test/testclass.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,9 @@
#include "fixture.h"
#include "helpers.h"
#include "settings.h"
#include "tokenize.h"
#include "tokenlist.h"

#include <cstddef>
#include <list>
#include <sstream>
#include <string>
#include <vector>

Expand Down Expand Up @@ -8703,9 +8700,7 @@ class TestClass : public TestFixture {
void checkUselessOverride_(const char* file, int line, const char code[]) {
const Settings settings = settingsBuilder().severity(Severity::style).build();

std::vector<std::string> files(1, "test.cpp");
Tokenizer tokenizer(settings, *this);
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
SimpleTokenizer2 tokenizer(settings, *this, code, "test.cpp");

ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);

Expand Down Expand Up @@ -9085,11 +9080,9 @@ class TestClass : public TestFixture {
// getFileInfo
std::list<Check::FileInfo*> fileInfo;
for (const std::string& c: code) {
Tokenizer tokenizer(settingsDefault, *this);
std::istringstream istr(c);
SimpleTokenizer tokenizer{settingsDefault, *this};
const std::string filename = std::to_string(fileInfo.size()) + ".cpp";
ASSERT(tokenizer.list.createTokens(istr, filename));
ASSERT(tokenizer.simplifyTokens1(""));
ASSERT(tokenizer.tokenize(c, filename));
fileInfo.push_back(check.getFileInfo(tokenizer, settingsDefault));
}

Expand Down Expand Up @@ -9151,9 +9144,7 @@ class TestClass : public TestFixture {
void checkReturnByReference_(const char* file, int line, const char (&code)[size]) {
const Settings settings = settingsBuilder().severity(Severity::performance).library("std.cfg").build();

std::vector<std::string> files(1, "test.cpp");
Tokenizer tokenizer(settings, *this);
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
SimpleTokenizer2 tokenizer(settings, *this, code, "test.cpp");

ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);

Expand Down
10 changes: 3 additions & 7 deletions test/testcondition.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
#include "helpers.h"
#include "platform.h"
#include "settings.h"
#include "tokenize.h"

#include <cstddef>
#include <limits>
Expand Down Expand Up @@ -139,9 +138,8 @@ class TestCondition : public TestFixture {
#define check(...) check_(__FILE__, __LINE__, __VA_ARGS__)
void check_(const char* file, int line, const char code[], const CheckOptions& options = make_default_obj()) {
const Settings settings = settingsBuilder(options.s ? *options.s : settings0).certainty(Certainty::inconclusive, options.inconclusive).build();
Tokenizer tokenizer(settings, *this);
std::vector<std::string> files(1, options.cpp ? "test.cpp" : "test.c");
PreprocessorHelper::preprocess(code, files, tokenizer, *this);

SimpleTokenizer2 tokenizer(settings, *this, code, options.cpp ? "test.cpp" : "test.c");

// Tokenizer..
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
Expand All @@ -155,9 +153,7 @@ class TestCondition : public TestFixture {
{
const Settings settings = settingsBuilder(settings0).severity(Severity::performance).certainty(Certainty::inconclusive).build();

std::vector<std::string> files(1, "test.cpp");
Tokenizer tokenizer(settings, *this);
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
SimpleTokenizer2 tokenizer(settings, *this, code, "test.cpp");

// Tokenizer..
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
Expand Down
5 changes: 1 addition & 4 deletions test/testincompletestatement.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
#include "helpers.h"
#include "settings.h"
#include "fixture.h"
#include "tokenize.h"

#include <string>
#include <vector>
Expand All @@ -44,9 +43,7 @@ class TestIncompleteStatement : public TestFixture {
void check_(const char* file, int line, const char code[], const CheckOptions& options = make_default_obj()) {
const Settings settings1 = settingsBuilder(settings).certainty(Certainty::inconclusive, options.inconclusive).build();

std::vector<std::string> files(1, options.cpp ? "test.cpp" : "test.c");
Tokenizer tokenizer(settings1, *this);
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
SimpleTokenizer2 tokenizer(settings1, *this, code, options.cpp ? "test.cpp" : "test.c");

// Tokenize..
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
Expand Down
5 changes: 1 addition & 4 deletions test/testleakautovar.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
#include "fixture.h"
#include "helpers.h"
#include "settings.h"
#include "tokenize.h"

#include <cstddef>
#include <string>
Expand Down Expand Up @@ -3195,9 +3194,7 @@ class TestLeakAutoVarRecursiveCountLimit : public TestFixture {

#define checkP(...) checkP_(__FILE__, __LINE__, __VA_ARGS__)
void checkP_(const char* file, int line, const char code[], bool cpp = false) {
std::vector<std::string> files(1, cpp?"test.cpp":"test.c");
Tokenizer tokenizer(settings, *this);
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
SimpleTokenizer2 tokenizer(settings, *this, code, cpp?"test.cpp":"test.c");

// Tokenizer..
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
Expand Down
5 changes: 1 addition & 4 deletions test/testnullpointer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
#include "library.h"
#include "settings.h"
#include "token.h"
#include "tokenize.h"

#include <cstddef>
#include <list>
Expand Down Expand Up @@ -201,9 +200,7 @@ class TestNullPointer : public TestFixture {
void checkP_(const char* file, int line, const char (&code)[size]) {
const Settings settings1 = settingsBuilder(settings).certainty(Certainty::inconclusive, false).build();

std::vector<std::string> files(1, "test.cpp");
Tokenizer tokenizer(settings1, *this);
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
SimpleTokenizer2 tokenizer(settings1, *this, code, "test.cpp");

// Tokenizer..
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
Expand Down
5 changes: 1 addition & 4 deletions test/testother.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
#include "platform.h"
#include "settings.h"
#include "standards.h"
#include "tokenize.h"

#include <cstddef>
#include <string>
Expand Down Expand Up @@ -354,9 +353,7 @@ class TestOther : public TestFixture {
settings->standards.cpp = Standards::CPPLatest;
settings->certainty.enable(Certainty::inconclusive);

std::vector<std::string> files(1, options.cpp ? "test.cpp" : "test.c");
Tokenizer tokenizer(*settings, *this);
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
SimpleTokenizer2 tokenizer(*settings, *this, code, options.cpp ? "test.cpp" : "test.c");

// Tokenizer..
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
Expand Down
Loading
Loading