Skip to content

Token: avoid unnecessary lookups in update_property_info() and added unit tests for it #7369

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
firewave merged 4 commits into from
Mar 18, 2025
Merged

Token: avoid unnecessary lookups in update_property_info() and added unit tests for it #7369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9b26d4a
TestToken: added test for `update_property_info()`
firewave Mar 11, 2025
5dbebf1
Token: only look up control-flow keywords in `update_property_info()`…
firewave Mar 11, 2025
750c6df
Token: only look up standard types in `update_property_info()` if nec…
firewave Mar 11, 2025
9e29597
Token: avoid always false control-flow lookup in `update_property_inf…
firewave Mar 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions lib/token.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,8 @@ static const std::unordered_set<std::string> controlFlowKeywords = {

void Token::update_property_info()
{
setFlag(fIsControlFlowKeyword, controlFlowKeywords.find(mStr) != controlFlowKeywords.end());
setFlag(fIsControlFlowKeyword, false);
// TODO: clear fIsLong
isStandardType(false);

if (!mStr.empty()) {
Expand All @@ -124,11 +125,21 @@ void Token::update_property_info()
else if (std::isalpha((unsigned char)mStr[0]) || mStr[0] == '_' || mStr[0] == '$') { // Name
if (mImpl->mVarId)
tokType(eVariable);
else if (mTokensFrontBack.list.isKeyword(mStr) || mStr == "asm") // TODO: not a keyword
else if (mTokensFrontBack.list.isKeyword(mStr)) {
tokType(eKeyword);
update_property_isStandardType();
if (mTokType != eType) // cannot be a control-flow keyword when it is a type
setFlag(fIsControlFlowKeyword, controlFlowKeywords.find(mStr) != controlFlowKeywords.end());
}
else if (mStr == "asm") { // TODO: not a keyword
tokType(eKeyword);
}
// TODO: remove condition? appears to be (no longer necessary) protection for reset of varids in Tokenizer::setVarId()
else if (mTokType != eVariable && mTokType != eFunction && mTokType != eType && mTokType != eKeyword)
else if (mTokType != eVariable && mTokType != eFunction && mTokType != eType && mTokType != eKeyword) {
tokType(eName);
// some types are not being treated as keywords
update_property_isStandardType();
}
} else if (simplecpp::Token::isNumberLike(mStr)) {
if ((MathLib::isInt(mStr) || MathLib::isFloat(mStr)) && mStr.find('_') == std::string::npos)
tokType(eNumber);
Expand All @@ -148,6 +159,7 @@ void Token::update_property_info()
mStr == "||" ||
mStr == "!"))
tokType(eLogicalOp);
// TODO: should link check only apply to < and >? Token::link() suggests so
else if (mStr.size() <= 2 && !mLink &&
(mStr == "==" ||
mStr == "!=" ||
Expand All @@ -168,11 +180,12 @@ void Token::update_property_info()
tokType(eEllipsis);
else
tokType(eOther);

update_property_isStandardType();
} else {
tokType(eNone);
}
// TODO: make sure varid is only set for eVariable
//assert(!mImpl->mVarId || mTokType == eVariable);
// TODO: validate type for linked token?
}

static const std::unordered_set<std::string> stdTypes = { "bool"
Expand Down
3 changes: 3 additions & 0 deletions lib/token.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -956,9 +956,12 @@ class CPPCHECKLIB Token {
return;

mImpl->mVarId = id;
// TODO: remove special handling?
if (id != 0) {
tokType(eVariable);
isStandardType(false);
// TODO: clear fIsLong
// TODO: clear fIsControlFlowKeyword
} else {
update_property_info();
}
Expand Down
251 changes: 251 additions & 0 deletions test/testtoken.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,14 @@ class TestToken : public TestFixture {

TEST_CASE(hasKnownIntValue);

TEST_CASE(update_property_info);
TEST_CASE(update_property_info_evariable);
TEST_CASE(update_property_info_ekeyword_c);
TEST_CASE(update_property_info_ekeyword_cpp);
TEST_CASE(update_property_info_ebracket_link);
TEST_CASE(update_property_info_ecomparisonop_link);
TEST_CASE(update_property_info_etype_c);
TEST_CASE(update_property_info_etype_cpp);
TEST_CASE(varid_reset);
}

Expand Down Expand Up @@ -1245,6 +1253,249 @@ class TestToken : public TestFixture {
_assert_tok(file, line, &tok, t, l, std, ctrl);
}

void update_property_info() const
{
assert_tok("", Token::Type::eNone);
assert_tok("true", Token::Type::eBoolean);
assert_tok("false", Token::Type::eBoolean);
assert_tok("\"\"", Token::Type::eString);
assert_tok("L\"\"", Token::Type::eString, /*l=*/ true);
assert_tok("'a'", Token::Type::eChar);
assert_tok("L'a'", Token::Type::eChar, /*l=*/ true);
// eVariable has separate test
// eKeyword with specific languages and standards has separate tests
assert_tok("sizeof", Token::Type::eKeyword);
assert_tok("goto", Token::Type::eKeyword, /*l=*/ false, /*std=*/ false, /*ctrl=*/ true);
assert_tok("asm", Token::Type::eKeyword);
assert_tok("a", Token::Type::eName);
assert_tok("_a", Token::Type::eName);
assert_tok("$a", Token::Type::eName);
assert_tok("bool2", Token::Type::eName);
assert_tok("0", Token::Type::eNumber);
assert_tok("-0", Token::Type::eNumber);
assert_tok("+0", Token::Type::eNumber);
assert_tok("0xa", Token::Type::eNumber);
assert_tok("010", Token::Type::eNumber);
assert_tok("0b0", Token::Type::eNumber);
assert_tok("0.0", Token::Type::eNumber);
assert_tok("0x0.3p10", Token::Type::eNumber);
assert_tok("0z", Token::Type::eNumber); // TODO: not a valid number
assert_tok("0_km", Token::Type::eName); // user literal
assert_tok("=", Token::Type::eAssignmentOp);
assert_tok("<<=", Token::Type::eAssignmentOp);
assert_tok(">>=", Token::Type::eAssignmentOp);
assert_tok("+=", Token::Type::eAssignmentOp);
assert_tok("-=", Token::Type::eAssignmentOp);
assert_tok("*=", Token::Type::eAssignmentOp);
assert_tok("/=", Token::Type::eAssignmentOp);
assert_tok("%=", Token::Type::eAssignmentOp);
assert_tok("&=", Token::Type::eAssignmentOp);
assert_tok("|=", Token::Type::eAssignmentOp);
assert_tok("^=", Token::Type::eAssignmentOp);
assert_tok(",", Token::Type::eExtendedOp);
assert_tok("[", Token::Type::eExtendedOp);
assert_tok("]", Token::Type::eExtendedOp);
assert_tok("(", Token::Type::eExtendedOp);
assert_tok(")", Token::Type::eExtendedOp);
assert_tok("?", Token::Type::eExtendedOp);
assert_tok(":", Token::Type::eExtendedOp);
assert_tok("<<", Token::Type::eArithmeticalOp);
assert_tok(">>", Token::Type::eArithmeticalOp);
assert_tok("+", Token::Type::eArithmeticalOp);
assert_tok("-", Token::Type::eArithmeticalOp);
assert_tok("*", Token::Type::eArithmeticalOp);
assert_tok("/", Token::Type::eArithmeticalOp);
assert_tok("%", Token::Type::eArithmeticalOp);
assert_tok("&", Token::Type::eBitOp);
assert_tok("|", Token::Type::eBitOp);
assert_tok("^", Token::Type::eBitOp);
assert_tok("~", Token::Type::eBitOp);
assert_tok("&&", Token::Type::eLogicalOp);
assert_tok("||", Token::Type::eLogicalOp);
assert_tok("!", Token::Type::eLogicalOp);
assert_tok("==", Token::Type::eComparisonOp);
assert_tok("!=", Token::Type::eComparisonOp);
assert_tok("<", Token::Type::eComparisonOp);
assert_tok("<=", Token::Type::eComparisonOp);
assert_tok(">", Token::Type::eComparisonOp);
assert_tok(">=", Token::Type::eComparisonOp);
// eComparisonOp with link has a separate test
assert_tok("<=>", Token::Type::eComparisonOp);
assert_tok("++", Token::Type::eIncDecOp);
assert_tok("--", Token::Type::eIncDecOp);
assert_tok("{", Token::Type::eBracket);
assert_tok("}", Token::Type::eBracket);
// < and > with link have a separate test
assert_tok("...", Token::Type::eEllipsis);
assert_tok(";", Token::Type::eOther);
// eType with specific languages and standards has separate tests
assert_tok("void", Token::Type::eType, /*l=*/ false, /*std=*/ true);
}

void update_property_info_evariable() const
{
{
TokensFrontBack tokensFrontBack(list);
Token tok(tokensFrontBack);
tok.str("var1");
tok.varId(17);
assert_tok(&tok, Token::Type::eVariable);
}
{
TokensFrontBack tokensFrontBack(list);
Token tok(tokensFrontBack);
tok.varId(17);
tok.str("var1");
assert_tok(&tok, Token::Type::eVariable);
}
}

void update_property_info_ekeyword_c() const
{
{
const Settings s = settingsBuilder().c(Standards::cstd_t::C89).build();
TokenList list_c{&s};
list_c.setLang(Standards::Language::C);
TokensFrontBack tokensFrontBack(list_c);
Token tok(tokensFrontBack);
tok.str("alignas"); // not a C89 keyword
assert_tok(&tok, Token::Type::eName);
}
{
TokenList list_c{&settingsDefault};
list_c.setLang(Standards::Language::C);
TokensFrontBack tokensFrontBack(list_c);
Token tok(tokensFrontBack);
tok.str("alignas"); // a C23 keyword
assert_tok(&tok, Token::Type::eKeyword);
}
{
TokenList list_c{&settingsDefault};
list_c.setLang(Standards::Language::C);
TokensFrontBack tokensFrontBack(list_c);
Token tok(tokensFrontBack);
tok.str("and_eq"); // a C++ keyword
assert_tok(&tok, Token::Type::eName);
}
}

void update_property_info_ekeyword_cpp() const
{
{
const Settings s = settingsBuilder().cpp(Standards::cppstd_t::CPP03).build();
TokenList list_cpp{&s};
list_cpp.setLang(Standards::Language::CPP);
TokensFrontBack tokensFrontBack(list_cpp);
Token tok(tokensFrontBack);
tok.str("consteval"); // not a C++03 keyword
assert_tok(&tok, Token::Type::eName);
}
{
TokenList list_cpp{&settingsDefault};
list_cpp.setLang(Standards::Language::CPP);
TokensFrontBack tokensFrontBack(list_cpp);
Token tok(tokensFrontBack);
tok.str("consteval"); // a C++20 keyword
assert_tok(&tok, Token::Type::eKeyword);
}
{
TokenList list_cpp{&settingsDefault};
list_cpp.setLang(Standards::Language::CPP);
TokensFrontBack tokensFrontBack(list_cpp);
Token tok(tokensFrontBack);
tok.str("typeof_unqual"); // a C keyword
assert_tok(&tok, Token::Type::eName);
}
}

void update_property_info_ebracket_link() const
{
{
TokensFrontBack tokensFrontBack(list);
Token tok(tokensFrontBack);
tok.str("<");

Token tok2(tokensFrontBack);
tok.link(&tok2);
assert_tok(&tok, Token::Type::eBracket);
}

{
TokensFrontBack tokensFrontBack(list);
Token tok(tokensFrontBack);

Token tok2(tokensFrontBack);
tok.link(&tok2);

tok.str("<");
assert_tok(&tok, Token::Type::eBracket);
}
}

void update_property_info_ecomparisonop_link() const
{
{
TokensFrontBack tokensFrontBack(list);
Token tok(tokensFrontBack);
tok.str("==");

Token tok2(tokensFrontBack);
tok.link(&tok2); // TODO: does not (and probably should not) update
assert_tok(&tok, Token::Type::eComparisonOp);
}

{
TokensFrontBack tokensFrontBack(list);
Token tok(tokensFrontBack);

Token tok2(tokensFrontBack);
tok.link(&tok2);

tok.str("==");
assert_tok(&tok, Token::Type::eOther); // TODO: this looks wrong
}
}

void update_property_info_etype_c() const
{
{
TokenList list_c{&settingsDefault};
list_c.setLang(Standards::Language::C);
TokensFrontBack tokensFrontBack(list_c);
Token tok(tokensFrontBack);
tok.str("char"); // not treated as keyword in TokenList::isKeyword()
assert_tok(&tok, Token::Type::eType, /*l=*/ false, /*std=*/ true);
}
{
TokenList list_c{&settingsDefault};
list_c.setLang(Standards::Language::C);
TokensFrontBack tokensFrontBack(list_c);
Token tok(tokensFrontBack);
tok.str("size_t"); // not treated as keyword in TokenList::isKeyword()
assert_tok(&tok, Token::Type::eType, /*l=*/ false, /*std=*/ true);
}
}

void update_property_info_etype_cpp() const
{
{
TokenList list_cpp{&settingsDefault};
list_cpp.setLang(Standards::Language::CPP);
TokensFrontBack tokensFrontBack(list_cpp);
Token tok(tokensFrontBack);
tok.str("bool"); // not treated as keyword in TokenList::isKeyword()
assert_tok(&tok, Token::Type::eType, /*l=*/ false, /*std=*/ true);
}
{
TokenList list_cpp{&settingsDefault};
list_cpp.setLang(Standards::Language::CPP);
TokensFrontBack tokensFrontBack(list_cpp);
Token tok(tokensFrontBack);
tok.str("size_t");
assert_tok(&tok, Token::Type::eType, /*l=*/ false, /*std=*/ true);
}
}

void varid_reset() const
{
TokenList list_c{&settingsDefault};
Expand Down
Loading