Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

9453 lines (8233 sloc) 343.754 kb
/*
* Cppcheck - A tool for static C/C++ code analysis
* Copyright (C) 2007-2012 Daniel Marjamäki and Cppcheck team.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
//---------------------------------------------------------------------------
#include "tokenize.h"
#include "mathlib.h"
#include "settings.h"
#include "check.h"
#include "path.h"
#include "symboldatabase.h"
#include "templatesimplifier.h"
#include "timer.h"
#include <cstring>
#include <sstream>
#include <cassert>
#include <cctype>
#include <stack>
//---------------------------------------------------------------------------
Tokenizer::Tokenizer() :
list(0),
_settings(0),
_errorLogger(0),
_symbolDatabase(0),
_varId(0),
_codeWithTemplates(false), //is there any templates?
m_timerResults(NULL)
{
}
Tokenizer::Tokenizer(const Settings *settings, ErrorLogger *errorLogger) :
list(settings),
_settings(settings),
_errorLogger(errorLogger),
_symbolDatabase(0),
_varId(0),
_codeWithTemplates(false), //is there any templates?
m_timerResults(NULL)
{
// make sure settings are specified
assert(_settings);
}
Tokenizer::~Tokenizer()
{
delete _symbolDatabase;
}
//---------------------------------------------------------------------------
// SizeOfType - gives the size of a type
//---------------------------------------------------------------------------
unsigned int Tokenizer::sizeOfType(const Token *type) const
{
if (!type || type->str().empty())
return 0;
if (type->type() == Token::eString)
return static_cast<unsigned int>(Token::getStrLength(type) + 1);
std::map<std::string, unsigned int>::const_iterator it = _typeSize.find(type->str());
if (it == _typeSize.end())
return 0;
else if (type->isLong()) {
if (type->str() == "double")
return _settings->sizeof_long_double;
else if (type->str() == "long")
return _settings->sizeof_long_long;
}
return it->second;
}
//---------------------------------------------------------------------------
Token *Tokenizer::copyTokens(Token *dest, const Token *first, const Token *last, bool one_line)
{
std::stack<Token *> links;
Token *tok2 = dest;
unsigned int linenrs = dest->linenr();
unsigned int commonFileIndex = dest->fileIndex();
for (const Token *tok = first; tok != last->next(); tok = tok->next()) {
tok2->insertToken(tok->str());
tok2 = tok2->next();
tok2->fileIndex(commonFileIndex);
tok2->linenr(linenrs);
tok2->type(tok->type());
tok2->isUnsigned(tok->isUnsigned());
tok2->isSigned(tok->isSigned());
tok2->isPointerCompare(tok->isPointerCompare());
tok2->isLong(tok->isLong());
tok2->isUnused(tok->isUnused());
tok2->setExpandedMacro(tok->isExpandedMacro());
tok2->varId(tok->varId());
// Check for links and fix them up
if (tok2->str() == "(" || tok2->str() == "[" || tok2->str() == "{")
links.push(tok2);
else if (tok2->str() == ")" || tok2->str() == "]" || tok2->str() == "}") {
if (links.empty())
return tok2;
Token * link = links.top();
tok2->link(link);
link->link(tok2);
links.pop();
}
if (!one_line && tok->next())
linenrs += tok->next()->linenr() - tok->linenr();
}
return tok2;
}
//---------------------------------------------------------------------------
void Tokenizer::duplicateTypedefError(const Token *tok1, const Token *tok2, const std::string &type) const
{
if (tok1 && !(_settings->isEnabled("style") && _settings->inconclusive))
return;
std::list<const Token*> locationList;
locationList.push_back(tok1);
locationList.push_back(tok2);
const std::string tok2_str = tok2 ? tok2->str() : std::string("name");
reportError(locationList, Severity::style, "variableHidingTypedef",
std::string("The " + type + " '" + tok2_str + "' hides a typedef with the same name."), true);
}
void Tokenizer::duplicateDeclarationError(const Token *tok1, const Token *tok2, const std::string &type) const
{
if (tok1 && !(_settings->isEnabled("style")))
return;
std::list<const Token*> locationList;
locationList.push_back(tok1);
locationList.push_back(tok2);
const std::string tok2_str = tok2 ? tok2->str() : std::string("name");
reportError(locationList, Severity::style, "unnecessaryForwardDeclaration",
std::string("The " + type + " '" + tok2_str + "' forward declaration is unnecessary. Type " + type + " is already declared earlier."));
}
// check if this statement is a duplicate definition
bool Tokenizer::duplicateTypedef(Token **tokPtr, const Token *name, const Token *typeDef, bool undefinedStruct) const
{
// check for an end of definition
const Token * tok = *tokPtr;
if (tok && Token::Match(tok->next(), ";|,|[|=|)|>|(|{")) {
const Token * end = tok->next();
if (end->str() == "[") {
end = end->link()->next();
} else if (end->str() == ",") {
// check for derived class
if (Token::Match(tok->previous(), "public|private|protected"))
return false;
// find end of definition
while (end && end->next() && !Token::Match(end->next(), ";|)|>")) {
if (end->next()->str() == "(")
end = end->linkAt(1);
end = end->next();
}
if (end)
end = end->next();
} else if (end->str() == "(") {
if (tok->previous()->str().find("operator") == 0) {
// conversion operator
return false;
} else if (tok->previous()->str() == "typedef") {
// typedef of function returning this type
return false;
} else if (Token::Match(tok->previous(), "public:|private:|protected:")) {
return false;
} else if (tok->previous()->str() == ">") {
if (!Token::Match(tok->tokAt(-2), "%type%"))
return false;
if (!Token::Match(tok->tokAt(-3), ",|<"))
return false;
duplicateTypedefError(*tokPtr, name, "template instantiation");
*tokPtr = end->link();
return true;
} else if (Token::Match(tok->previous(), "%type%")) {
if (end->link()->next()->str() == "{") {
duplicateTypedefError(*tokPtr, name, "function");
*tokPtr = end->link()->next()->link();
return true;
}
}
}
if (end) {
if (Token::simpleMatch(end, ") {")) { // function parameter ?
// look backwards
if (Token::Match(tok->previous(), "%type%") &&
!Token::Match(tok->previous(), "return|new|const")) {
duplicateTypedefError(*tokPtr, name, "function parameter");
// duplicate definition so skip entire function
*tokPtr = end->next()->link();
return true;
}
} else if (end->str() == ">") { // template parameter ?
// look backwards
if (Token::Match(tok->previous(), "%type%") &&
!Token::Match(tok->previous(), "return|new|const|volatile")) {
// duplicate definition so skip entire template
while (end && end->str() != "{")
end = end->next();
if (end) {
duplicateTypedefError(*tokPtr, name, "template parameter");
*tokPtr = end->link();
return true;
}
}
} else {
// look backwards
if (Token::Match(tok->previous(), "typedef|}|>") ||
(tok->previous()->str() == "*" && tok->next()->str() != "(") ||
(Token::Match(tok->previous(), "%type%") &&
(!Token::Match(tok->previous(), "return|new|const|friend|public|private|protected|throw|extern") &&
!Token::simpleMatch(tok->tokAt(-2), "friend class")))) {
// scan backwards for the end of the previous statement
while (tok && tok->previous() && !Token::Match(tok->previous(), ";|{")) {
if (tok->previous()->str() == "}") {
tok = tok->previous()->link();
} else if (tok->previous()->str() == "typedef") {
duplicateTypedefError(*tokPtr, name, "typedef");
return true;
} else if (tok->previous()->str() == "enum") {
duplicateTypedefError(*tokPtr, name, "enum");
return true;
} else if (tok->previous()->str() == "struct") {
if (tok->strAt(-2) == "typedef" &&
tok->next()->str() == "{" &&
typeDef->strAt(3) != "{") {
// declaration after forward declaration
return true;
} else if (tok->next()->str() == "{") {
if (!undefinedStruct)
duplicateTypedefError(*tokPtr, name, "struct");
return true;
} else if (Token::Match(tok->next(), ")|*")) {
return true;
} else if (tok->next()->str() == name->str()) {
return true;
} else if (tok->next()->str() != ";") {
duplicateTypedefError(*tokPtr, name, "struct");
return true;
} else {
// forward declaration after declaration
duplicateDeclarationError(*tokPtr, name, "struct");
return false;
}
} else if (tok->previous()->str() == "union") {
if (tok->next()->str() != ";") {
duplicateTypedefError(*tokPtr, name, "union");
return true;
} else {
// forward declaration after declaration
duplicateDeclarationError(*tokPtr, name, "union");
return false;
}
} else if (tok->previous()->str() == "class") {
if (tok->next()->str() != ";") {
duplicateTypedefError(*tokPtr, name, "class");
return true;
} else {
// forward declaration after declaration
duplicateDeclarationError(*tokPtr, name, "class");
return false;
}
}
tok = tok->previous();
}
duplicateTypedefError(*tokPtr, name, "variable");
return true;
}
}
}
}
return false;
}
void Tokenizer::unsupportedTypedef(const Token *tok) const
{
if (!_settings->debugwarnings)
return;
std::ostringstream str;
const Token *tok1 = tok;
unsigned int level = 0;
while (tok) {
if (level == 0 && tok->str() == ";")
break;
else if (tok->str() == "{")
++level;
else if (tok->str() == "}") {
if (!level)
break;
--level;
}
if (tok != tok1)
str << " ";
str << tok->str();
tok = tok->next();
}
if (tok)
str << " ;";
reportError(tok1, Severity::debug, "debug",
"Failed to parse \'" + str.str() + "\'. The checking continues anyway.");
}
Token * Tokenizer::deleteInvalidTypedef(Token *typeDef)
{
Token *tok = NULL;
// remove typedef but leave ;
while (typeDef->next()) {
if (typeDef->next()->str() == ";") {
typeDef->deleteNext();
break;
} else if (typeDef->next()->str() == "{")
Token::eraseTokens(typeDef, typeDef->linkAt(1));
else if (typeDef->next()->str() == "}")
break;
typeDef->deleteNext();
}
if (typeDef != list.front()) {
tok = typeDef->previous();
tok->deleteNext();
} else {
list.front()->deleteThis();
tok = list.front();
}
return tok;
}
struct Space {
std::string className;
const Token * classEnd;
bool isNamespace;
};
static Token *splitDefinitionFromTypedef(Token *tok)
{
Token *tok1;
std::string name;
bool isConst = false;
if (tok->next()->str() == "const") {
tok->deleteNext();
isConst = true;
}
if (tok->strAt(2) == "{") { // unnamed
tok1 = tok->linkAt(2);
if (tok1 && tok1->next()) {
// use typedef name if available
if (Token::Match(tok1->next(), "%type%"))
name = tok1->next()->str();
else { // create a unique name
static unsigned int count = 0;
name = "Unnamed" + MathLib::longToString(count++);
}
tok->next()->insertToken(name);
} else
return NULL;
} else if (tok->strAt(3) == ":") {
tok1 = tok->tokAt(4);
while (tok1 && tok1->str() != "{")
tok1 = tok1->next();
if (!tok1)
return NULL;
tok1 = tok1->link();
name = tok->strAt(2);
} else { // has a name
tok1 = tok->linkAt(3);
if (!tok1)
return NULL;
name = tok->strAt(2);
}
tok1->insertToken(";");
tok1 = tok1->next();
if (tok1->next() && tok1->next()->str() == ";" && tok1->previous()->str() == "}") {
tok->deleteThis();
tok1->deleteThis();
return NULL;
} else {
tok1->insertToken("typedef");
tok1 = tok1->next();
Token * tok3 = tok1;
if (isConst) {
tok1->insertToken("const");
tok1 = tok1->next();
}
tok1->insertToken(tok->next()->str()); // struct, union or enum
tok1 = tok1->next();
tok1->insertToken(name);
tok->deleteThis();
tok = tok3;
}
return tok;
}
/* This function is called when processing function related typedefs.
* If simplifyTypedef generates an "Internal Error" message and the
* code that generated it deals in some way with functions, then this
* function will probably need to be extended to handle a new function
* related pattern */
static Token *processFunc(Token *tok2, bool inOperator)
{
if (tok2->next() && tok2->next()->str() != ")" &&
tok2->next()->str() != ",") {
// skip over tokens for some types of canonicalization
if (Token::Match(tok2->next(), "( * %type% ) ("))
tok2 = tok2->linkAt(5);
else if (Token::Match(tok2->next(), "* ( * %type% ) ("))
tok2 = tok2->linkAt(6);
else if (Token::Match(tok2->next(), "* ( * %type% ) ;"))
tok2 = tok2->tokAt(5);
else if (Token::Match(tok2->next(), "* ( %type% [") &&
Token::Match(tok2->linkAt(4), "] ) ;|="))
tok2 = tok2->linkAt(4)->next();
else if (Token::Match(tok2->next(), "* ( * %type% ("))
tok2 = tok2->linkAt(5)->next();
else if (Token::simpleMatch(tok2->next(), "* [") &&
Token::simpleMatch(tok2->linkAt(2), "] ;"))
tok2 = tok2->next();
else {
if (tok2->next()->str() == "(")
tok2 = tok2->next()->link();
else if (!inOperator && !Token::Match(tok2->next(), "[|>|;")) {
tok2 = tok2->next();
while (Token::Match(tok2, "*|&") &&
!Token::Match(tok2->next(), ")|>"))
tok2 = tok2->next();
// skip over namespace
while (Token::Match(tok2, "%var% ::"))
tok2 = tok2->tokAt(2);
if (!tok2)
return NULL;
if (tok2->str() == "(" &&
tok2->link()->next()->str() == "(") {
tok2 = tok2->link();
if (tok2->next()->str() == "(")
tok2 = tok2->next()->link();
}
// skip over typedef parameter
if (tok2->next() && tok2->next()->str() == "(") {
tok2 = tok2->next()->link();
if (tok2->next()->str() == "(")
tok2 = tok2->next()->link();
}
}
}
}
return tok2;
}
void Tokenizer::simplifyTypedef()
{
std::vector<Space> spaceInfo;
bool isNamespace = false;
std::string className;
bool hasClass = false;
bool goback = false;
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (_errorLogger && !list.getFiles().empty())
_errorLogger->reportProgress(list.getFiles()[0], "Tokenize (typedef)", tok->progressValue());
if (goback) {
//jump back once, see the comment at the end of the function
goback = false;
tok = tok->previous();
}
// Skip typedefs inside parentheses (#2453 and #4002)
if (tok->str() == "(" && tok->strAt(1) == "typedef") {
tok = tok->next();
continue;
}
if (Token::Match(tok, "class|struct|namespace %any%") &&
(!tok->previous() || (tok->previous() && tok->previous()->str() != "enum"))) {
isNamespace = (tok->str() == "namespace");
hasClass = true;
className = tok->next()->str();
continue;
} else if (hasClass && tok->str() == ";") {
hasClass = false;
continue;
} else if (hasClass && tok->str() == "{") {
Space info;
info.isNamespace = isNamespace;
info.className = className;
info.classEnd = tok->link();
spaceInfo.push_back(info);
hasClass = false;
continue;
} else if (!spaceInfo.empty() && tok->str() == "}" && spaceInfo.back().classEnd == tok) {
spaceInfo.pop_back();
continue;
} else if (tok->str() != "typedef")
continue;
// pull struct, union, enum or class definition out of typedef
// use typedef name for unnamed struct, union, enum or class
if (Token::Match(tok->next(), "const| struct|enum|union|class %type% {") ||
Token::Match(tok->next(), "const| struct|enum|union|class {")) {
Token *tok1 = splitDefinitionFromTypedef(tok);
if (!tok1)
continue;
tok = tok1;
} else if (Token::Match(tok->next(), "const| struct|class %type% :")) {
Token *tok1 = tok;
while (tok1 && tok1->str() != ";" && tok1->str() != "{")
tok1 = tok1->next();
if (tok1 && tok1->str() == "{") {
tok1 = splitDefinitionFromTypedef(tok);
if (!tok1)
continue;
tok = tok1;
}
}
/** @todo add support for union */
bool undefinedStruct = false;
if (Token::Match(tok, "typedef enum|struct %type% %type% ;") && tok->strAt(2) == tok->strAt(3)) {
if (tok->next()->str() == "enum") {
tok->deleteNext(3);
tok->deleteThis();
if (tok->next())
tok->deleteThis();
//now the next token to process is 'tok', not 'tok->next()';
goback = true;
continue;
} else {
const std::string pattern("struct " + tok->strAt(2) + " {|:");
const Token *tok2 = Token::findmatch(list.front(), pattern.c_str(), tok);
if (!tok2)
undefinedStruct = true;
}
}
Token *typeName;
std::list<std::string> pointers;
Token *typeStart = 0;
Token *typeEnd = 0;
Token *argStart = 0;
Token *argEnd = 0;
Token *arrayStart = 0;
Token *arrayEnd = 0;
Token *specStart = 0;
Token *specEnd = 0;
Token *typeDef = tok;
Token *argFuncRetStart = 0;
Token *argFuncRetEnd = 0;
Token *funcStart = 0;
Token *funcEnd = 0;
Token *tokOffset = tok->next();
bool function = false;
bool functionPtr = false;
bool functionRef = false;
bool functionRetFuncPtr = false;
bool functionPtrRetFuncPtr = false;
bool ptrToArray = false;
bool refToArray = false;
bool ptrMember = false;
bool typeOf = false;
Token *namespaceStart = 0;
Token *namespaceEnd = 0;
// check for invalid input
if (!tok->next()) {
syntaxError(tok);
return;
}
if (tok->next()->str() == "::" || Token::Match(tok->next(), "%type%")) {
typeStart = tok->next();
while (Token::Match(tokOffset, "const|signed|unsigned|struct|enum %type%") ||
(tokOffset->next() && tokOffset->next()->isStandardType()))
tokOffset = tokOffset->next();
typeEnd = tokOffset;
tokOffset = tokOffset->next();
bool atEnd = false;
while (!atEnd) {
if (tokOffset && tokOffset->str() == "::") {
typeEnd = tokOffset;
tokOffset = tokOffset->next();
}
if (Token::Match(tokOffset, "%type%") &&
tokOffset->next() && !Token::Match(tokOffset->next(), "[|;|,|(")) {
typeEnd = tokOffset;
tokOffset = tokOffset->next();
} else if (Token::simpleMatch(tokOffset, "const (")) {
typeEnd = tokOffset;
tokOffset = tokOffset->next();
atEnd = true;
} else
atEnd = true;
}
} else
continue; // invalid input
// check for invalid input
if (!tokOffset) {
syntaxError(tok);
return;
}
// check for template
if (tokOffset->str() == "<") {
tokOffset->findClosingBracket(typeEnd);
while (typeEnd && Token::Match(typeEnd->next(), ":: %type%"))
typeEnd = typeEnd->tokAt(2);
if (!typeEnd) {
// internal error
return;
}
while (Token::Match(typeEnd->next(), "const|volatile"))
typeEnd = typeEnd->next();
tok = typeEnd;
tokOffset = tok->next();
}
// check for pointers and references
while (Token::Match(tokOffset, "*|&|const")) {
pointers.push_back(tokOffset->str());
tokOffset = tokOffset->next();
}
// check for invalid input
if (!tokOffset) {
syntaxError(tok);
return;
}
if (Token::Match(tokOffset, "%type%")) {
// found the type name
typeName = tokOffset;
tokOffset = tokOffset->next();
// check for array
if (tokOffset && tokOffset->str() == "[") {
arrayStart = tokOffset;
bool atEnd = false;
while (!atEnd) {
while (tokOffset->next() && !Token::Match(tokOffset->next(), ";|,")) {
tokOffset = tokOffset->next();
}
if (!tokOffset->next())
return; // invalid input
else if (tokOffset->next()->str() == ";")
atEnd = true;
else if (tokOffset->str() == "]")
atEnd = true;
else
tokOffset = tokOffset->next();
}
arrayEnd = tokOffset;
tokOffset = tokOffset->next();
}
// check for end or another
if (Token::Match(tokOffset, ";|,"))
tok = tokOffset;
// or a function typedef
else if (tokOffset && tokOffset->str() == "(") {
// unhandled typedef, skip it and continue
if (typeName->str() == "void") {
unsupportedTypedef(typeDef);
tok = deleteInvalidTypedef(typeDef);
if (tok == list.front())
//now the next token to process is 'tok', not 'tok->next()';
goback = true;
continue;
}
// unhandled function pointer, skip it and continue
// TODO: handle such typedefs. See ticket #3314
else if (Token::Match(tokOffset, "( %type% ::") &&
Token::Match(tokOffset->link()->tokAt(-3), ":: * %var% ) (")) {
unsupportedTypedef(typeDef);
tok = deleteInvalidTypedef(typeDef);
if (tok == list.front())
//now the next token to process is 'tok', not 'tok->next()';
goback = true;
continue;
}
// function pointer
else if (Token::Match(tokOffset, "( * %var% ) (")) {
// name token wasn't a name, it was part of the type
typeEnd = typeEnd->next();
functionPtr = true;
tokOffset = tokOffset->next();
funcStart = tokOffset;
funcEnd = tokOffset;
tokOffset = tokOffset->tokAt(3);
typeName = tokOffset->tokAt(-2);
argStart = tokOffset;
argEnd = tokOffset->link();
tok = argEnd->next();
}
// function
else if (Token::Match(tokOffset->link(), ") const| ;|,")) {
function = true;
if (tokOffset->link()->next()->str() == "const") {
specStart = tokOffset->link()->next();
specEnd = specStart;
}
argStart = tokOffset;
argEnd = tokOffset->link();
tok = argEnd->next();
if (specStart)
tok = tok->next();
}
// syntax error
else {
syntaxError(tok);
return;
}
}
// unhandled typedef, skip it and continue
else {
unsupportedTypedef(typeDef);
tok = deleteInvalidTypedef(typeDef);
if (tok == list.front())
//now the next token to process is 'tok', not 'tok->next()';
goback = true;
continue;
}
}
// typeof: typedef __typeof__ ( ... ) type;
else if (Token::simpleMatch(tokOffset->previous(), "__typeof__ (") &&
Token::Match(tokOffset->link(), ") %type% ;")) {
argStart = tokOffset;
argEnd = tokOffset->link();
typeName = tokOffset->link()->next();
tok = typeName->next();
typeOf = true;
}
// function: typedef ... ( .... type )( ... );
// typedef ... (( .... type )( ... ));
// typedef ... ( * ( .... type )( ... ));
else if ((tokOffset->str() == "(" &&
Token::Match(tokOffset->link()->previous(), "%type% ) (") &&
Token::Match(tokOffset->link()->next()->link(), ") const|volatile|;")) ||
(Token::simpleMatch(tokOffset, "( (") &&
Token::Match(tokOffset->next()->link()->previous(), "%type% ) (") &&
Token::Match(tokOffset->next()->link()->next()->link(), ") const|volatile| ) ;|,")) ||
(Token::simpleMatch(tokOffset, "( * (") &&
Token::Match(tokOffset->linkAt(2)->previous(), "%type% ) (") &&
Token::Match(tokOffset->linkAt(2)->next()->link(), ") const|volatile| ) ;|,"))) {
if (tokOffset->next()->str() == "(")
tokOffset = tokOffset->next();
else if (Token::simpleMatch(tokOffset, "( * (")) {
pointers.push_back("*");
tokOffset = tokOffset->tokAt(2);
}
if (tokOffset->link()->strAt(-2) == "*")
functionPtr = true;
else
function = true;
funcStart = tokOffset->next();
tokOffset = tokOffset->link();
funcEnd = tokOffset->tokAt(-2);
typeName = tokOffset->previous();
argStart = tokOffset->next();
argEnd = tokOffset->next()->link();
tok = argEnd->next();
Token *spec = tok;
if (Token::Match(spec, "const|volatile")) {
specStart = spec;
specEnd = spec;
while (Token::Match(spec->next(), "const|volatile")) {
specEnd = spec->next();
spec = specEnd;
}
tok = specEnd->next();
}
if (tok->str() == ")")
tok = tok->next();
}
else if (Token::Match(tokOffset, "( %type% (")) {
function = true;
if (tokOffset->link()->next()) {
tok = tokOffset->link()->next();
tokOffset = tokOffset->tokAt(2);
typeName = tokOffset->previous();
argStart = tokOffset;
argEnd = tokOffset->link();
} else {
// internal error
continue;
}
}
// pointer to function returning pointer to function
else if (Token::Match(tokOffset, "( * ( * %type% ) (") &&
Token::simpleMatch(tokOffset->linkAt(6), ") ) (") &&
Token::Match(tokOffset->linkAt(6)->linkAt(2), ") ;|,")) {
functionPtrRetFuncPtr = true;
tokOffset = tokOffset->tokAt(6);
typeName = tokOffset->tokAt(-2);
argStart = tokOffset;
argEnd = tokOffset->link();
argFuncRetStart = argEnd->tokAt(2);
argFuncRetEnd = argFuncRetStart->link();
tok = argFuncRetEnd->next();
}
// function returning pointer to function
else if (Token::Match(tokOffset, "( * %type% (") &&
Token::simpleMatch(tokOffset->linkAt(3), ") ) (") &&
Token::Match(tokOffset->linkAt(3)->linkAt(2), ") ;|,")) {
functionRetFuncPtr = true;
tokOffset = tokOffset->tokAt(3);
typeName = tokOffset->previous();
argStart = tokOffset;
argEnd = tokOffset->link();
argFuncRetStart = argEnd->tokAt(2);
argFuncRetEnd = argFuncRetStart->link();
tok = argFuncRetEnd->next();
} else if (Token::Match(tokOffset, "( * ( %type% ) (")) {
functionRetFuncPtr = true;
tokOffset = tokOffset->tokAt(5);
typeName = tokOffset->tokAt(-2);
argStart = tokOffset;
argEnd = tokOffset->link();
argFuncRetStart = argEnd->tokAt(2);
argFuncRetEnd = argFuncRetStart->link();
tok = argFuncRetEnd->next();
}
// pointer/reference to array
else if (Token::Match(tokOffset, "( *|& %type% ) [")) {
ptrToArray = (tokOffset->next()->str() == "*");
refToArray = (tokOffset->next()->str() == "&");
tokOffset = tokOffset->tokAt(2);
typeName = tokOffset;
arrayStart = tokOffset->tokAt(2);
arrayEnd = arrayStart->link();
tok = arrayEnd->next();
}
// pointer to class member
else if (Token::Match(tokOffset, "( %type% :: * %type% ) ;")) {
tokOffset = tokOffset->tokAt(2);
namespaceStart = tokOffset->previous();
namespaceEnd = tokOffset;
ptrMember = true;
tokOffset = tokOffset->tokAt(2);
typeName = tokOffset;
tok = tokOffset->tokAt(2);
}
// unhandled typedef, skip it and continue
else {
unsupportedTypedef(typeDef);
tok = deleteInvalidTypedef(typeDef);
if (tok == list.front())
//now the next token to process is 'tok', not 'tok->next()';
goback = true;
continue;
}
bool done = false;
bool ok = true;
while (!done) {
std::string pattern = typeName->str();
int scope = 0;
bool inScope = true;
bool exitThisScope = false;
int exitScope = 0;
bool simplifyType = false;
bool inMemberFunc = false;
int memberScope = 0;
bool globalScope = false;
std::size_t classLevel = spaceInfo.size();
for (Token *tok2 = tok; tok2; tok2 = tok2->next()) {
// check for end of scope
if (tok2->str() == "}") {
// check for end of member function
if (inMemberFunc) {
--memberScope;
if (memberScope == 0)
inMemberFunc = false;
}
if (classLevel > 0 && tok2 == spaceInfo[classLevel - 1].classEnd) {
--classLevel;
pattern.clear();
for (std::size_t i = classLevel; i < spaceInfo.size(); ++i)
pattern += (spaceInfo[i].className + " :: ");
pattern += typeName->str();
} else {
--scope;
if (scope < 0)
inScope = false;
if (exitThisScope) {
if (scope < exitScope)
exitThisScope = false;
}
}
}
// check for operator typedef
/** @todo add support for multi-token operators */
else if (tok2->str() == "operator" &&
tok2->next()->str() == typeName->str() &&
tok2->strAt(2) == "(" &&
Token::Match(tok2->linkAt(2), ") const| {")) {
// check for qualifier
if (tok2->previous()->str() == "::") {
// check for available and matching class name
if (!spaceInfo.empty() && classLevel < spaceInfo.size() &&
tok2->strAt(-2) == spaceInfo[classLevel].className) {
tok2 = tok2->next();
simplifyType = true;
}
}
}
// check for member functions
else if (Token::Match(tok2, ") const| {")) {
const Token *func = tok2->link()->previous();
if (!func)
continue;
if (func->previous()) { // Ticket #4239
/** @todo add support for multi-token operators */
if (func->previous()->str() == "operator")
func = func->previous();
// check for qualifier
if (func->previous()->str() == "::") {
// check for available and matching class name
if (!spaceInfo.empty() && classLevel < spaceInfo.size() &&
func->strAt(-2) == spaceInfo[classLevel].className) {
memberScope = 0;
inMemberFunc = true;
}
}
}
}
// check for entering a new namespace
else if (Token::Match(tok2, "namespace %any% {")) {
if (classLevel < spaceInfo.size() &&
spaceInfo[classLevel].isNamespace &&
spaceInfo[classLevel].className == tok2->next()->str()) {
++classLevel;
pattern.clear();
for (std::size_t i = classLevel; i < spaceInfo.size(); ++i)
pattern += (spaceInfo[i].className + " :: ");
pattern += typeName->str();
}
++scope;
}
// check for entering a new scope
else if (tok2->str() == "{") {
// keep track of scopes within member function
if (inMemberFunc)
++memberScope;
++scope;
}
// check for typedef that can be substituted
else if (Token::Match(tok2, pattern.c_str()) ||
(inMemberFunc && tok2->str() == typeName->str())) {
std::string pattern1;
// member function class variables don't need qualification
if (inMemberFunc && tok2->str() == typeName->str())
pattern1 = tok2->str();
else
pattern1 = pattern;
if (pattern1.find("::") != std::string::npos) { // has a "something ::"
if (tok2->strAt(-1) == "::") {
tok2->tokAt(-2)->deleteNext();
globalScope = true;
}
for (std::size_t i = classLevel; i < spaceInfo.size(); ++i) {
tok2->deleteNext(2);
}
simplifyType = true;
} else if ((inScope && !exitThisScope) || inMemberFunc) {
if (tok2->strAt(-1) == "::") {
// Don't replace this typename if it's preceded by "::" unless it's a namespace
if (!spaceInfo.empty() && (tok2->strAt(-2) == spaceInfo[0].className) && spaceInfo[0].isNamespace) {
tok2->tokAt(-3)->deleteNext(2);
simplifyType = true;
}
} else if (Token::Match(tok2->previous(), "case %type% :")) {
tok2 = tok2->next();
} else if (duplicateTypedef(&tok2, typeName, typeDef, undefinedStruct)) {
exitScope = scope;
// skip to end of scope if not already there
if (tok2->str() != "}") {
while (tok2->next()) {
if (tok2->next()->str() == "{")
tok2 = tok2->linkAt(1)->previous();
else if (tok2->next()->str() == "}")
break;
tok2 = tok2->next();
}
}
} else if (tok2->previous()->str() != ".") {
simplifyType = true;
}
}
}
if (simplifyType) {
// can't simplify 'operator functionPtr ()' and 'functionPtr operator ... ()'
if (functionPtr && (tok2->previous()->str() == "operator" ||
tok2->next()->str() == "operator")) {
simplifyType = false;
tok2 = tok2->next();
break;
}
// There are 2 categories of typedef substitutions:
// 1. variable declarations that preserve the variable name like
// global, local, and function parameters
// 2. not variable declarations that have no name like derived
// classes, casts, operators, and template parameters
// try to determine which category this substitution is
bool inCast = false;
bool inTemplate = false;
bool inOperator = false;
bool inSizeof = false;
// check for derived class: class A : some_typedef {
bool isDerived = Token::Match(tok2->previous(), "public|protected|private %type% {|,");
// check for cast: (some_typedef) A or static_cast<some_typedef>(A)
// todo: check for more complicated casts like: (const some_typedef *)A
if ((tok2->previous()->str() == "(" && tok2->next()->str() == ")" && tok2->strAt(-2) != "sizeof") ||
(tok2->previous()->str() == "<" && Token::simpleMatch(tok2->next(), "> (")))
inCast = true;
// check for template parameters: t<some_typedef> t1
else if (Token::Match(tok2->previous(), "<|,") &&
Token::Match(tok2->next(), "&|*| &|*| >|,"))
inTemplate = true;
else if (Token::Match(tok2->tokAt(-2), "sizeof ( %type% )"))
inSizeof = true;
// check for operator
if (tok2->strAt(-1) == "operator" ||
Token::simpleMatch(tok2->tokAt(-2), "operator const"))
inOperator = true;
// skip over class or struct in derived class declaration
bool structRemoved = false;
if (isDerived && Token::Match(typeStart, "class|struct")) {
if (typeStart->str() == "struct")
structRemoved = true;
typeStart = typeStart->next();
}
// start substituting at the typedef name by replacing it with the type
tok2->str(typeStart->str());
// restore qualification if it was removed
if (typeStart->str() == "struct" || structRemoved) {
if (structRemoved)
tok2 = tok2->previous();
if (globalScope) {
tok2->insertToken("::");
tok2 = tok2->next();
}
for (std::size_t i = classLevel; i < spaceInfo.size(); ++i) {
tok2->insertToken(spaceInfo[i].className);
tok2 = tok2->next();
tok2->insertToken("::");
tok2 = tok2->next();
}
}
// add remainder of type
tok2 = copyTokens(tok2, typeStart->next(), typeEnd);
if (!pointers.empty()) {
std::list<std::string>::const_iterator iter;
for (iter = pointers.begin(); iter != pointers.end(); ++iter) {
tok2->insertToken(*iter);
tok2 = tok2->next();
}
}
if (funcStart && funcEnd) {
tok2->insertToken("(");
tok2 = tok2->next();
Token *tok3 = tok2;
tok2 = copyTokens(tok2, funcStart, funcEnd);
if (!inCast)
tok2 = processFunc(tok2, inOperator);
if (!tok2)
break;
tok2->insertToken(")");
tok2 = tok2->next();
Token::createMutualLinks(tok2, tok3);
tok2 = copyTokens(tok2, argStart, argEnd);
if (specStart) {
Token *spec = specStart;
tok2->insertToken(spec->str());
tok2 = tok2->next();
while (spec != specEnd) {
spec = spec->next();
tok2->insertToken(spec->str());
tok2 = tok2->next();
}
}
}
else if (functionPtr || functionRef || function) {
// don't add parenthesis around function names because it
// confuses other simplifications
bool needParen = true;
if (!inTemplate && function && tok2->next() && tok2->next()->str() != "*")
needParen = false;
if (needParen) {
tok2->insertToken("(");
tok2 = tok2->next();
}
Token *tok3 = tok2;
if (namespaceStart) {
const Token *tok4 = namespaceStart;
while (tok4 != namespaceEnd) {
tok2->insertToken(tok4->str());
tok2 = tok2->next();
tok4 = tok4->next();
}
tok2->insertToken(namespaceEnd->str());
tok2 = tok2->next();
}
if (functionPtr) {
tok2->insertToken("*");
tok2 = tok2->next();
} else if (functionRef) {
tok2->insertToken("&");
tok2 = tok2->next();
}
if (!inCast)
tok2 = processFunc(tok2, inOperator);
if (needParen) {
tok2->insertToken(")");
tok2 = tok2->next();
Token::createMutualLinks(tok2, tok3);
}
tok2 = copyTokens(tok2, argStart, argEnd);
if (inTemplate)
tok2 = tok2->next();
if (specStart) {
Token *spec = specStart;
tok2->insertToken(spec->str());
tok2 = tok2->next();
while (spec != specEnd) {
spec = spec->next();
tok2->insertToken(spec->str());
tok2 = tok2->next();
}
}
} else if (functionRetFuncPtr || functionPtrRetFuncPtr) {
tok2->insertToken("(");
tok2 = tok2->next();
Token *tok3 = tok2;
tok2->insertToken("*");
tok2 = tok2->next();
Token * tok4 = 0;
if (functionPtrRetFuncPtr) {
tok2->insertToken("(");
tok2 = tok2->next();
tok4 = tok2;
tok2->insertToken("*");
tok2 = tok2->next();
}
// skip over variable name if there
if (!inCast) {
if (tok2->next()->str() != ")")
tok2 = tok2->next();
}
if (tok4 && functionPtrRetFuncPtr) {
tok2->insertToken(")");
tok2 = tok2->next();
Token::createMutualLinks(tok2, tok4);
}
tok2 = copyTokens(tok2, argStart, argEnd);
tok2->insertToken(")");
tok2 = tok2->next();
Token::createMutualLinks(tok2, tok3);
tok2 = copyTokens(tok2, argFuncRetStart, argFuncRetEnd);
} else if (ptrToArray || refToArray) {
tok2->insertToken("(");
tok2 = tok2->next();
Token *tok3 = tok2;
if (ptrToArray)
tok2->insertToken("*");
else
tok2->insertToken("&");
tok2 = tok2->next();
// skip over name
if (tok2->next()->str() != ")") {
if (tok2->next()->str() != "(")
tok2 = tok2->next();
// check for function and skip over args
if (tok2->next()->str() == "(")
tok2 = tok2->next()->link();
// check for array
if (tok2->next()->str() == "[")
tok2 = tok2->next()->link();
} else {
// syntax error
}
tok2->insertToken(")");
Token::createMutualLinks(tok2->next(), tok3);
} else if (ptrMember) {
if (Token::simpleMatch(tok2, "* (")) {
tok2->insertToken("*");
tok2 = tok2->next();
} else {
tok2->insertToken("(");
tok2 = tok2->next();
Token *tok3 = tok2;
const Token *tok4 = namespaceStart;
while (tok4 != namespaceEnd) {
tok2->insertToken(tok4->str());
tok2 = tok2->next();
tok4 = tok4->next();
}
tok2->insertToken(namespaceEnd->str());
tok2 = tok2->next();
tok2->insertToken("*");
tok2 = tok2->next();
// skip over name
tok2 = tok2->next();
tok2->insertToken(")");
tok2 = tok2->next();
Token::createMutualLinks(tok2, tok3);
}
} else if (typeOf) {
tok2 = copyTokens(tok2, argStart, argEnd);
} else if (tok2->tokAt(2) && tok2->strAt(2) == "[") {
while (tok2->tokAt(2) && tok2->strAt(2) == "[")
tok2 = tok2->linkAt(2)->previous();
}
if (arrayStart && arrayEnd) {
do {
if (!tok2->next()) {
syntaxError(tok2);
return; // can't recover so quit
}
if (!inCast && !inSizeof)
tok2 = tok2->next();
// reference to array?
if (tok2->str() == "&") {
tok2 = tok2->previous();
tok2->insertToken("(");
Token *tok3 = tok2->next();
// handle missing variable name
if (tok2->strAt(3) == ")" || tok2->strAt(3) == ",")
tok2 = tok2->tokAt(2);
else
tok2 = tok2->tokAt(3);
tok2->insertToken(")");
tok2 = tok2->next();
Token::createMutualLinks(tok2, tok3);
}
if (!tok2->next()) {
syntaxError(tok2);
return; // can't recover so quit
}
tok2 = copyTokens(tok2, arrayStart, arrayEnd);
tok2 = tok2->next();
if (tok2->str() == "=") {
if (tok2->next()->str() == "{")
tok2 = tok2->next()->link()->next();
else if (tok2->next()->str().at(0) == '\"')
tok2 = tok2->tokAt(2);
}
} while (Token::Match(tok2, ", %var% ;|'|=|,"));
}
simplifyType = false;
}
}
if (tok->str() == ";")
done = true;
else if (tok->str() == ",") {
arrayStart = 0;
arrayEnd = 0;
tokOffset = tok->next();
pointers.clear();
while (Token::Match(tokOffset, "*|&")) {
pointers.push_back(tokOffset->str());
tokOffset = tokOffset->next();
}
if (Token::Match(tokOffset, "%type%")) {
typeName = tokOffset;
tokOffset = tokOffset->next();
if (tokOffset && tokOffset->str() == "[") {
arrayStart = tokOffset;
bool atEnd = false;
while (!atEnd) {
while (tokOffset->next() && !Token::Match(tokOffset->next(), ";|,"))
tokOffset = tokOffset->next();
if (!tokOffset->next())
return; // invalid input
else if (tokOffset->next()->str() == ";")
atEnd = true;
else if (tokOffset->str() == "]")
atEnd = true;
else
tokOffset = tokOffset->next();
}
arrayEnd = tokOffset;
tokOffset = tokOffset->next();
}
if (Token::Match(tokOffset, ";|,"))
tok = tokOffset;
else {
// we encountered a typedef we don't support yet so just continue
done = true;
ok = false;
}
} else {
// we encountered a typedef we don't support yet so just continue
done = true;
ok = false;
}
} else {
// something is really wrong (internal error)
done = true;
ok = false;
}
}
if (ok) {
// remove typedef
Token::eraseTokens(typeDef, tok);
if (typeDef != list.front()) {
tok = typeDef->previous();
tok->deleteNext();
//no need to remove last token in the list
if (tok->tokAt(2))
tok->deleteNext();
} else {
list.front()->deleteThis();
//no need to remove last token in the list
if (list.front()->next())
list.front()->deleteThis();
tok = list.front();
//now the next token to process is 'tok', not 'tok->next()';
goback = true;
}
}
}
}
void Tokenizer::simplifyMulAndParens()
{
for (Token *tok = list.front()->tokAt(3); tok; tok = tok->next()) {
if (tok->isName()) {
//fix ticket #2784 - improved by ticket #3184
unsigned int closedpars = 0;
Token *tokend = tok->next();
Token *tokbegin = tok->previous();
while (tokend && tokend->str() == ")") {
++closedpars;
tokend = tokend->next();
}
if (!tokend || !(tokend->isAssignmentOp()))
continue;
while (tokbegin && (tokbegin->str() == "&" || tokbegin->str() == "(")) {
if (tokbegin->str() == "&") {
if (Token::Match(tokbegin->tokAt(-2), "[;{}&(] *")) {
//remove '* &'
tokbegin = tokbegin->tokAt(-2);
tokbegin->deleteNext(2);
} else if (Token::Match(tokbegin->tokAt(-3), "[;{}&(] * (")) {
if (!closedpars)
break;
--closedpars;
//remove ')'
tok->deleteNext();
//remove '* ( &'
tokbegin = tokbegin->tokAt(-3);
tokbegin->deleteNext(3);
} else
break;
} else if (tokbegin->str() == "(") {
if (!closedpars)
break;
//find consecutive opening parentheses
unsigned int openpars = 0;
while (tokbegin && tokbegin->str() == "(" && openpars <= closedpars) {
++openpars;
tokbegin = tokbegin->previous();
}
if (!tokbegin || openpars > closedpars)
break;
if ((openpars == closedpars && Token::Match(tokbegin, "[;{}]")) ||
Token::Match(tokbegin->tokAt(-2), "[;{}&(] * &") ||
Token::Match(tokbegin->tokAt(-3), "[;{}&(] * ( &")) {
//remove the excessive parentheses around the variable
while (openpars--) {
tok->deleteNext();
tokbegin->deleteNext();
--closedpars;
}
} else
break;
}
}
}
}
}
bool Tokenizer::tokenize(std::istream &code,
const char FileName[],
const std::string &configuration,
const bool preprocessorCondition)
{
// make sure settings specified
assert(_settings);
// Fill the map _typeSize..
_typeSize.clear();
_typeSize["char"] = 1;
_typeSize["bool"] = _settings->sizeof_bool;
_typeSize["short"] = _settings->sizeof_short;
_typeSize["int"] = _settings->sizeof_int;
_typeSize["long"] = _settings->sizeof_long;
_typeSize["float"] = _settings->sizeof_float;
_typeSize["double"] = _settings->sizeof_double;
_typeSize["size_t"] = _settings->sizeof_size_t;
_typeSize["*"] = _settings->sizeof_pointer;
_configuration = configuration;
if (!list.createTokens(code, Path::getRelativePath(Path::simplifyPath(FileName), _settings->_basePaths))) {
cppcheckError(0);
return false;
}
// if MACRO
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "if|for|while|BOOST_FOREACH %var% (")) {
if (Token::simpleMatch(tok, "for each"))
// 'for each ( )' -> 'for ( )'
tok->deleteNext();
else {
// locate the ')' parenthesis (the Token::link is not set yet)
// Then replace 'if MACRO(X)' with 'if (MACRO(X))'
unsigned int parlevel = 0;
for (Token *tok2 = tok; tok2; tok2 = tok2->next()) {
if (tok2->str() == "(")
++parlevel;
else if (tok2->str() == ")") {
if (parlevel == 1) {
tok->insertToken("(");
tok2->insertToken(")");
}
if (parlevel <= 1)
break;
--parlevel;
}
}
}
}
}
// Combine wide strings
for (Token *tok = list.front(); tok; tok = tok->next()) {
while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) {
// Combine 'L "string"'
tok->str(tok->next()->str());
tok->deleteNext();
tok->isLong(true);
}
}
// Combine strings
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str()[0] != '"')
continue;
tok->str(simplifyString(tok->str()));
while (tok->next() && tok->next()->type() == Token::eString) {
tok->next()->str(simplifyString(tok->next()->str()));
// Two strings after each other, combine them
tok->concatStr(tok->next()->str());
tok->deleteNext();
}
}
// replace inline SQL with "asm()" (Oracle PRO*C). Ticket: #1959
simplifySQL();
// Simplify JAVA/C# code
if (isJavaOrCSharp())
simplifyJavaAndCSharp();
// Concatenate double sharp: 'a ## b' -> 'ab'
concatenateDoubleSharp();
if (!createLinks()) {
// Source has syntax errors, can't proceed
return false;
}
// replace __LINE__ macro with line number
simplifyLineMacro();
// replace 'NULL' and similar '0'-defined macros with '0'
simplifyNull();
// replace 'sin(0)' to '0' and other similar math expressions
simplifyMathExpressions();
// combine "- %num%"
concatenateNegativeNumber();
// simplify simple calculations
for (Token *tok = list.front() ? list.front()->next() : NULL; tok; tok = tok->next()) {
if (tok->isNumber())
TemplateSimplifier::simplifyNumericCalculations(tok->previous());
}
// remove extern "C" and extern "C" {}
if (isCPP())
simplifyExternC();
// simplify weird but legal code: "[;{}] ( { code; } ) ;"->"[;{}] code;"
simplifyRoundCurlyParenthesis();
// check for simple syntax errors..
for (const Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::simpleMatch(tok, "> struct {") &&
Token::simpleMatch(tok->linkAt(2), "} ;")) {
syntaxError(tok);
list.deallocateTokens();
return false;
}
}
simplifyDoWhileAddBraces();
if (!simplifyIfAddBraces())
return false;
// Combine tokens..
for (Token *tok = list.front(); tok && tok->next(); tok = tok->next()) {
const char c1 = tok->str()[0];
if (tok->str().length() == 1 && tok->next()->str().length() == 1) {
const char c2 = tok->next()->str()[0];
// combine +-*/ and =
if (c2 == '=' && (strchr("+-*/%&|^=!<>", c1))) {
tok->str(tok->str() + c2);
tok->deleteNext();
continue;
}
// replace "->" with "."
else if (c1 == '-' && c2 == '>') {
tok->str(".");
tok->deleteNext();
continue;
}
}
else if (tok->str() == ">>" && tok->next()->str() == "=") {
tok->str(">>=");
tok->deleteNext();
}
else if (tok->str() == "<<" && tok->next()->str() == "=") {
tok->str("<<=");
tok->deleteNext();
}
else if ((c1 == 'p' || c1 == '_') && tok->next()->str() == ":" && tok->strAt(2) != ":") {
if (tok->str() == "private" || tok->str() == "protected" || tok->str() == "public" || tok->str() == "__published") {
tok->str(tok->str() + ":");
tok->deleteNext();
continue;
}
}
}
// Simplify: 0[foo] -> *(foo)
for (Token* tok = list.front(); tok; tok = tok->next()) {
if (Token::simpleMatch(tok, "0 [") && tok->linkAt(1)) {
tok->str("*");
tok->next()->str("(");
tok->linkAt(1)->str(")");
}
}
// Remove "volatile", "inline", "register", and "restrict"
simplifyKeyword();
// Convert K&R function declarations to modern C
simplifyVarDecl(true);
if (!simplifyFunctionParameters())
return false;
// specify array size..
arraySize();
// simplify labels and 'case|default'-like syntaxes
simplifyLabelsCaseDefault();
// simplify '[;{}] * & ( %any% ) =' to '%any% ='
simplifyMulAndParens();
// ";a+=b;" => ";a=a+b;"
simplifyCompoundAssignment();
if (!preprocessorCondition) {
if (hasComplicatedSyntaxErrorsInTemplates()) {
list.deallocateTokens();
return false;
}
}
simplifyDefaultAndDeleteInsideClass();
// Remove __declspec()
simplifyDeclspec();
// remove some unhandled macros in global scope
removeMacrosInGlobalScope();
// remove calling conventions __cdecl, __stdcall..
simplifyCallingConvention();
// remove __attribute__((?))
simplifyAttribute();
// remove unnecessary member qualification..
removeUnnecessaryQualification();
// Add std:: in front of std classes, when using namespace std; was given
simplifyNamespaceStd();
// remove Microsoft MFC..
simplifyMicrosoftMFC();
// convert Microsoft memory functions
simplifyMicrosoftMemoryFunctions();
// convert Microsoft string functions
simplifyMicrosoftStringFunctions();
// Remove Qt signals and slots
simplifyQtSignalsSlots();
// remove Borland stuff..
simplifyBorland();
// Remove __builtin_expect, likely and unlikely
simplifyBuiltinExpect();
if (hasEnumsWithTypedef()) {
// #2449: syntax error: enum with typedef in it
list.deallocateTokens();
return false;
}
simplifyDebugNew();
// typedef..
if (m_timerResults) {
Timer t("Tokenizer::tokenize::simplifyTypedef", _settings->_showtime, m_timerResults);
simplifyTypedef();
} else {
simplifyTypedef();
}
for (Token* tok = list.front(); tok;) {
if (Token::Match(tok, "union|struct|class union|struct|class"))
tok->deleteNext();
else
tok = tok->next();
}
// catch bad typedef canonicalization
//
// to reproduce bad typedef, download upx-ucl from:
// http://packages.debian.org/sid/upx-ucl
// analyse the file src/stub/src/i386-linux.elf.interp-main.c
if (!validate()) {
// Source has syntax errors, can't proceed
return false;
}
// enum..
simplifyEnum();
// Remove __asm..
simplifyAsm();
// When the assembly code has been cleaned up, no @ is allowed
for (const Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str() == "(")
tok = tok->link();
else if (tok->str()[0] == '@') {
list.deallocateTokens();
return false;
}
}
// convert platform dependent types to standard types
// 32 bits: size_t -> unsigned long
// 64 bits: size_t -> unsigned long long
simplifyPlatformTypes();
// collapse compound standard types into a single token
// unsigned long long int => long _isUnsigned=true,_isLong=true
simplifyStdType();
// simplify bit fields..
simplifyBitfields();
// Use "<" comparison instead of ">"
simplifyComparisonOrder();
// Simplify '(p == 0)' to '(!p)'
simplifyIfNot();
simplifyIfNotNull();
//simplify for: move out start-statement "for (a;b;c);" => "{ a; for(;b;c); }"
//not enabled because it fails many tests with testrunner.
//@todo fix these fails before enabling this simplification
/*for (Token* tok = list.front(); tok; tok = tok->next()) {
if (tok->str() == "(" && ( !tok->previous() || tok->previous()->str() != "for")) {
tok = tok->link();
continue;
}
if (!Token::Match(tok->previous(),"[{};] for ("))
continue;
//find the two needed semicolons inside the 'for'
const Token *firstsemicolon = Token::findsimplematch(tok->next(), ";", tok->next()->link());
if (!firstsemicolon)
continue;
const Token *secondsemicolon = Token::findsimplematch(firstsemicolon->next(), ";", tok->next()->link());
if (!secondsemicolon)
continue;
if (Token::findsimplematch(secondsemicolon->next(), ";", tok->next()->link()))
continue; //no more than two semicolons!
if (!tok->next()->link()->next())
continue; //there should be always something after 'for (...)'
Token *fortok = tok;
Token *begin = tok->tokAt(2);
Token *end = tok->next()->link();
if ( begin->str() != ";" ) {
tok = tok->previous();
tok->insertToken(";");
tok->insertToken("{");
tok = tok->next();
if (end->next()->str() =="{") {
end = end->next()->link();
end->insertToken("}");
Token::createMutualLinks(tok, end->next());
end = end->link()->previous();
} else {
if (end->next()->str() != ";")
end->insertToken(";");
end = end->next();
end->insertToken("}");
Token::createMutualLinks(tok, end->next());
}
end = firstsemicolon->previous();
Token::move(begin, end, tok);
tok = fortok;
end = fortok->next()->link();
}
//every 'for' is changed to 'for(;b;c), now it's possible to convert the 'for' to a 'while'.
//precisely, 'for(;b;c){code}'-> 'while(b){code + c;}'
fortok->str("while");
begin = firstsemicolon->previous();
begin->deleteNext();
begin = secondsemicolon->previous();
begin->deleteNext();
begin = begin->next();
if (begin->str() == ")") { //'for(;b;)' -> 'while(b)'
if (begin->previous()->str() == "(") //'for(;;)' -> 'while(true)'
begin->previous()->insertToken("true");
tok = fortok;
continue;
}
if (end->next()->str() =="{") {
tok = end->next()->link()->previous();
tok->insertToken(";");
} else {
tok = end;
if (end->next()->str() != ";")
tok->insertToken(";");
tok->insertToken("{");
tok = tok->tokAt(2);
tok->insertToken("}");
Token::createMutualLinks(tok->previous(), tok->next());
tok = tok->previous();
}
end = end->previous();
Token::move(begin, end, tok);
tok = fortok;
}*/
simplifyConst();
// struct simplification "struct S {} s; => struct S { } ; S s ;
simplifyStructDecl();
// struct initialization (must be used before simplifyVarDecl)
simplifyStructInit();
// Change initialisation of variable to assignment
simplifyInitVar();
// Split up variable declarations.
simplifyVarDecl(false);
// specify array size.. needed when arrays are split
arraySize();
// f(x=g()) => x=g(); f(x)
simplifyAssignmentInFunctionCall();
// x = ({ 123; }); => { x = 123; }
simplifyAssignmentBlock();
simplifyVariableMultipleAssign();
// Remove redundant parentheses
simplifyRedundantParenthesis();
for (Token *tok = list.front(); tok; tok = tok->next())
while (TemplateSimplifier::simplifyNumericCalculations(tok));
// Handle templates..
simplifyTemplates();
// Simplify templates.. sometimes the "simplifyTemplates" fail and
// then unsimplified function calls etc remain. These have the
// "wrong" syntax. So this function will just fix so that the
// syntax is corrected.
TemplateSimplifier::cleanupAfterSimplify(list.front());
// Simplify the operator "?:"
simplifyConditionOperator();
// remove exception specifications..
removeExceptionSpecifications();
// Collapse operator name tokens into single token
// operator = => operator=
simplifyOperatorName();
// Simplify pointer to standard types (C only)
simplifyPointerToStandardType();
// simplify function pointers
simplifyFunctionPointers();
// "if (not p)" => "if (!p)"
// "if (p and q)" => "if (p && q)"
// "if (p or q)" => "if (p || q)"
while (simplifyLogicalOperators()) { }
// Change initialisation of variable to assignment
simplifyInitVar();
// Split up variable declarations.
simplifyVarDecl(false);
if (!preprocessorCondition) {
if (m_timerResults) {
Timer t("Tokenizer::tokenize::setVarId", _settings->_showtime, m_timerResults);
setVarId();
} else {
setVarId();
}
createLinks2();
// Change initialisation of variable to assignment
simplifyInitVar();
}
// Convert e.g. atol("0") into 0
simplifyMathFunctions();
simplifyDoublePlusAndDoubleMinus();
simplifyArrayAccessSyntax();
list.front()->assignProgressValues();
removeRedundantSemicolons();
simplifyParameterVoid();
simplifyRedundantConsecutiveBraces();
simplifyEmptyNamespaces();
bool valid = validate();
if (valid)
createSymbolDatabase();
return valid;
}
//---------------------------------------------------------------------------
bool Tokenizer::hasComplicatedSyntaxErrorsInTemplates()
{
const Token *tok = TemplateSimplifier::hasComplicatedSyntaxErrorsInTemplates(list.front());
if (tok) {
syntaxError(tok);
return true;
}
return false;
}
void Tokenizer::simplifyDefaultAndDeleteInsideClass()
{
if (isC())
return;
// Remove "= default|delete" inside class|struct definitions
// Todo: Remove it if it is used "externally" too.
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "struct|class %var% :|{")) {
for (Token *tok2 = tok->tokAt(3); tok2; tok2 = tok2->next()) {
if (tok2->str() == "{")
tok2 = tok2->link();
else if (tok2->str() == "}")
break;
else if (Token::Match(tok2, ") = delete|default ;")) {
Token * const end = tok2->tokAt(4);
tok2 = tok2->link()->previous();
// operator ==|>|<|..
if (Token::Match(tok2->previous(), "operator %any%"))
tok2 = tok2->previous();
else if (Token::simpleMatch(tok2->tokAt(-2), "operator [ ]"))
tok2 = tok2->tokAt(-2);
else if (Token::simpleMatch(tok2->tokAt(-2), "operator ( )"))
tok2 = tok2->tokAt(-2);
else if (Token::simpleMatch(tok2->tokAt(-3), "operator delete [ ]"))
tok2 = tok2->tokAt(-3);
while ((tok2->isName() && tok2->str().find(":") == std::string::npos) ||
Token::Match(tok2, "[&*~]"))
tok2 = tok2->previous();
if (Token::Match(tok2, "[;{}]") || tok2->isName())
Token::eraseTokens(tok2, end);
else
tok2 = end->previous();
}
}
}
}
}
bool Tokenizer::hasEnumsWithTypedef()
{
for (const Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "enum %var% {")) {
tok = tok->tokAt(2);
const Token *tok2 = Token::findsimplematch(tok, "typedef", tok->link());
if (tok2) {
syntaxError(tok2);
return true;
}
}
}
return false;
}
void Tokenizer::concatenateDoubleSharp()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
// TODO: pattern should be "%var%|%num% ## %var%|%num%"
while (Token::Match(tok, "%any% ## %any%") &&
(tok->isName() || tok->isNumber()) &&
(tok->tokAt(2)->isName() || tok->tokAt(2)->isNumber())) {
tok->str(tok->str() + tok->strAt(2));
tok->deleteNext(2);
}
}
}
void Tokenizer::simplifyLineMacro()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str() == "__LINE__")
tok->str(MathLib::longToString(tok->linenr()));
}
}
void Tokenizer::simplifyNull()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str() == "NULL" || tok->str() == "__null" ||
tok->str() == "'\\0'" || tok->str() == "'\\x0'") {
tok->str("0");
} else if (tok->isNumber() &&
MathLib::isInt(tok->str()) &&
MathLib::toLongNumber(tok->str()) == 0) {
tok->str("0");
}
}
// nullptr..
if (isCPP() && _settings->standards.cpp == Standards::CPP11) {
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str() == "nullptr")
tok->str("0");
}
}
}
void Tokenizer::concatenateNegativeNumber()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "?|:|,|(|[|=|return|case|sizeof|%op% - %num%")) {
tok->deleteNext();
tok->next()->str("-" + tok->next()->str());
}
}
}
void Tokenizer::simplifyExternC()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "extern \"C\" {|")) {
if (tok->strAt(2) == "{") {
tok->linkAt(2)->deleteThis();
tok->deleteNext(2);
} else
tok->deleteNext();
tok->deleteThis();
}
}
}
void Tokenizer::simplifyRoundCurlyParenthesis()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
while (Token::Match(tok, "[;{}] ( {") &&
Token::simpleMatch(tok->linkAt(2), "} ) ;")) {
tok->linkAt(2)->previous()->deleteNext(3);
tok->deleteNext(2);
}
}
}
void Tokenizer::simplifySQL()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::simpleMatch(tok, "EXEC SQL")) {
const Token *end = tok->tokAt(2);
while (end && end->str() != ";")
end = end->next();
std::string instruction = tok->stringifyList(end);
// delete all tokens until ';'
Token::eraseTokens(tok, end);
// insert "asm ( "instruction" ) ;"
tok->str("asm");
// it can happen that 'end' is NULL when wrong code is inserted
if (!tok->next())
tok->insertToken(";");
tok->insertToken(")");
tok->insertToken("\"" + instruction + "\"");
tok->insertToken("(");
// jump to ';' and continue
tok = tok->tokAt(3);
}
}
}
void Tokenizer::simplifyDebugNew()
{
// convert Microsoft DEBUG_NEW macro to new
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str() == "DEBUG_NEW")
tok->str("new");
}
}
void Tokenizer::simplifyArrayAccessSyntax()
{
// 0[a] -> a[0]
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "%num% [ %var% ]")) {
const std::string temp = tok->str();
tok->str(tok->strAt(2));
tok->tokAt(2)->str(temp);
}
}
}
void Tokenizer::simplifyParameterVoid()
{
for (Token* tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "%var% ( void )"))
tok->next()->deleteNext();
}
}
void Tokenizer::simplifyRedundantConsecutiveBraces()
{
// Remove redundant consecutive braces, i.e. '.. { { .. } } ..' -> '.. { .. } ..'.
for (Token *tok = list.front(); tok;) {
if (Token::simpleMatch(tok, "= {")) {
tok = tok->linkAt(1);
} else if (Token::simpleMatch(tok, "{ {") && Token::simpleMatch(tok->next()->link(), "} }")) {
//remove internal parentheses
tok->next()->link()->deleteThis();
tok->deleteNext();
} else
tok = tok->next();
}
}
void Tokenizer::simplifyDoublePlusAndDoubleMinus()
{
// Convert + + into + and + - into -
for (Token *tok = list.front(); tok; tok = tok->next()) {
while (tok->next()) {
if (tok->str() == "+") {
if (tok->next()->str() == "+") {
tok->deleteNext();
continue;
} else if (tok->next()->str()[0] == '-') {
tok = tok->next();
if (tok->str().size() == 1) {
tok = tok->previous();
tok->str("-");
tok->deleteNext();
} else if (tok->isNumber()) {
tok->str(tok->str().substr(1));
tok = tok->previous();
tok->str("-");
}
continue;
}
} else if (tok->str() == "-") {
if (tok->next()->str() == "+") {
tok->deleteNext();
continue;
} else if (tok->next()->str()[0] == '-') {
tok = tok->next();
if (tok->str().size() == 1) {
tok = tok->previous();
tok->str("+");
tok->deleteNext();
} else if (tok->isNumber()) {
tok->str(tok->str().substr(1));
tok = tok->previous();
tok->str("+");
}
continue;
}
}
break;
}
}
}
void Tokenizer::simplifyJavaAndCSharp()
{
// better don't call isJava in the loop
const bool isJava_ = isJava();
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str() == "private")
tok->str("private:");
else if (tok->str() == "protected")
tok->str("protected:");
else if (tok->str() == "public")
tok->str("public:");
else if (isJava_) {
if (Token::Match(tok, ") throws %var% {"))
tok->deleteNext(2);
} else {
//remove 'using var;' from code
if (Token::Match(tok, "using %var% ;") &&
(!tok->previous() || Token::Match(tok->previous(), "[,;{}]"))) {
tok->deleteNext(2);
tok->deleteThis();
}
//simplify C# arrays of arrays and multidimension arrays
while (Token::Match(tok, "%type% [ ,|]") &&
(!tok->previous() || Token::Match(tok->previous(), "[,;{}]"))) {
Token *tok2 = tok->tokAt(2);
unsigned int count = 1;
while (tok2 && tok2->str() == ",") {
++count;
tok2 = tok2->next();
}
if (!tok2 || tok2->str() != "]")
break;
tok2 = tok2->next();
while (Token::Match(tok2, "[ ,|]")) {
tok2 = tok2->next();
while (tok2 && tok2->str() == ",") {
++count;
tok2 = tok2->next();
}
if (!tok2 || tok2->str() != "]")
break;
++count;
tok2 = tok2->next();
}
if (!tok2)
break;
else if (Token::Match(tok2, "%var% [;,=]")) {
Token::eraseTokens(tok, tok2);
do {
tok->insertToken("*");
} while (--count);
tok = tok2->tokAt(2);
}
}
if (!tok)
break;
}
}
}
/** Specify array size if it hasn't been given */
void Tokenizer::arraySize()
{
bool addlength = false;
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "%var% [ ] = { %str% } ;")) {
Token *t = tok->tokAt(3);
t->deleteNext();
t->next()->deleteNext();
addlength = true;
}
if (addlength || Token::Match(tok, "%var% [ ] = %str% ;")) {
tok = tok->next();
std::size_t sz = tok->strAt(3).length() - 1;
tok->insertToken(MathLib::longToString((unsigned int)sz));
addlength = false;
tok = tok->tokAt(5);
}
else if (Token::Match(tok, "%var% [ ] = {")) {
unsigned int sz = 1;
tok = tok->next();
Token *end = tok->linkAt(3);
for (Token *tok2 = tok->tokAt(4); tok2 && tok2 != end; tok2 = tok2->next()) {
if (tok2->str() == "{" || tok2->str() == "(" || tok2->str() == "[")
tok2 = tok2->link();
else if (tok2->str() == "<") { // Bailout. TODO: When link() supports <>, this bailout becomes unnecessary
sz = 0;
break;
} else if (tok2->str() == ",") {
if (!Token::Match(tok2->next(), "[},]"))
++sz;
else {
tok2 = tok2->previous();
tok2->deleteNext();
}
}
}
if (sz != 0)
tok->insertToken(MathLib::longToString(sz));
tok = end->next() ? end->next() : end;
}
}
}
/** simplify labels and case|default in the code: add a ";" if not already in.*/
void Tokenizer::simplifyLabelsCaseDefault()
{
bool executablescope = false;
unsigned int indentlevel = 0;
for (Token *tok = list.front(); tok; tok = tok->next()) {
// Simplify labels in the executable scope..
if (Token::Match(tok, ") const| {")) {
tok = tok->next();
if (tok->str() == "const")
tok = tok->next();
executablescope = true;
}
if (!executablescope)
continue;
if (tok->str() == "{") {
if (tok->previous()->str() == "=")
tok = tok->link();
else
++indentlevel;
} else if (tok->str() == "}") {
--indentlevel;
if (!indentlevel) {
executablescope = false;
continue;
}
} else if (tok->str() == "(" || tok->str() == "[")
tok = tok->link();
if (Token::Match(tok, "[;{}] case")) {
while (NULL != (tok = tok->next())) {
if (tok->str() == ":")
break;
}
if (!tok)
break;
else if (tok->str() == ":" &&
(!tok->next() || tok->next()->str() != ";")) {
tok->insertToken(";");
}
} else if (Token::Match(tok, "[;{}] %var% : !!;")) {
tok = tok->tokAt(2);
tok->insertToken(";");
}
}
}
void Tokenizer::simplifyTemplates()
{
if (isC())
return;
for (Token *tok = list.front(); tok; tok = tok->next()) {
// #2648 - simple fix for sizeof used as template parameter
// TODO: this is a bit hardcoded. make a bit more generic
if (Token::Match(tok, "%var% < sizeof ( %type% ) >") && tok->tokAt(4)->isStandardType()) {
Token * const tok3 = tok->next();
const unsigned int sizeOfResult = sizeOfType(tok3->tokAt(3));
tok3->deleteNext(4);
tok3->insertToken(MathLib::longToString(sizeOfResult));
}
}
TemplateSimplifier::simplifyTemplates(
list,
*_errorLogger,
_settings,
_codeWithTemplates);
}
//---------------------------------------------------------------------------
static bool setVarIdParseDeclaration(const Token **tok, const std::map<std::string,unsigned int> &variableId, bool executableScope)
{
const Token *tok2 = *tok;
bool ref = false;
if (!tok2->isName())
return false;
unsigned int typeCount = 0;
bool hasstruct = false; // Is there a "struct" or "class"?
while (tok2) {
if (tok2->isName()) {
if (tok2->str() == "class" || tok2->str() == "struct" || tok2->str() == "union") {
hasstruct = true;
typeCount = 0;
} else if (tok2->str() == "const") {
; // just skip "const"
} else if (!hasstruct && variableId.find(tok2->str()) != variableId.end() && tok2->previous()->str() != "::") {
++typeCount;
tok2 = tok2->next();
if (!tok2 || tok2->str() != "::")
break;
} else {
++typeCount;
}
} else if (tok2->str() == "<" && TemplateSimplifier::templateParameters(tok2) > 0) {
bool ok = tok2->findClosingBracket(tok2);
if (!ok || !tok2)
break;
} else if (tok2->str() == "&") {
ref = true;
} else if (tok2->str() != "*" && tok2->str() != "::") {
break;
}
tok2 = tok2->next();
}
if (tok2) {
*tok = tok2;
// In executable scopes, references must be assigned
// Catching by reference is an exception
if (executableScope && ref) {
if (tok2->str() == "(" || tok2->str() == "=")
; // reference is assigned => ok
else if (tok2->str() != ")" || !Token::simpleMatch(tok2->link()->previous(), "catch"))
return false; // not catching by reference => not declaration
}
}
// Check if array declaration is valid (#2638)
// invalid declaration: AAA a[4] = 0;
if (typeCount >= 2 && tok2 && tok2->str() == "[") {
const Token *tok3 = tok2;
while (tok3 && tok3->str() == "[") {
tok3 = tok3->link()->next();
}
if (Token::Match(tok3, "= %num%"))
return false;
}
return bool(typeCount >= 2 && tok2 && Token::Match(tok2->tokAt(-2), "!!:: %type%"));
}
static void setVarIdStructMembers(Token **tok1,
std::map<unsigned int, std::map<std::string,unsigned int> > *structMembers,
unsigned int *_varId)
{
Token *tok = *tok1;
while (Token::Match(tok->next(), ". %var% !!(")) {
const unsigned int struct_varid = tok->varId();
tok = tok->tokAt(2);
if (struct_varid == 0)
continue;
// Don't set varid for template function
if (TemplateSimplifier::templateParameters(tok->next()) > 0)
break;
std::map<std::string,unsigned int>& members = (*structMembers)[struct_varid];
if (members.empty() || members.find(tok->str()) == members.end()) {
members[tok->str()] = ++(*_varId);
tok->varId(*_varId);
} else {
tok->varId(members[tok->str()]);
}
}
if (tok)
*tok1 = tok;
}
static void setVarIdClassDeclaration(Token * const startToken,
const std::map<std::string, unsigned int> &variableId,
const unsigned int scopeStartVarId,
std::map<unsigned int, std::map<std::string,unsigned int> > *structMembers,
unsigned int *_varId)
{
// end of scope
const Token * const endToken = startToken->link();
// determine class name
std::string className;
for (const Token *tok = startToken->previous(); tok; tok = tok->previous()) {
if (!tok->isName() && tok->str() != ":")
break;
if (Token::Match(tok, "class|struct %type% [:{]")) {
className = tok->next()->str();
break;
}
}
// replace varids..
unsigned int indentlevel = 0;
bool initList = false;
for (Token *tok = startToken->next(); tok != endToken; tok = tok->next()) {
if (tok->str() == "{") {
initList = false;
++indentlevel;
} else if (tok->str() == "}")
--indentlevel;
else if (tok->isName() && tok->varId() <= scopeStartVarId) {
if (indentlevel > 0 || (initList && indentlevel == 0 && (tok->strAt(-1) == "," || tok->strAt(-1) == ":"))) {
if (Token::Match(tok->previous(), "::|."))
continue;
if (tok->next()->str() == "::") {
if (tok->str() == className)
tok = tok->tokAt(2);
else
continue;
}
const std::map<std::string, unsigned int>::const_iterator it = variableId.find(tok->str());
if (it != variableId.end()) {
tok->varId(it->second);
setVarIdStructMembers(&tok, structMembers, _varId);
}
}
} else if (indentlevel == 0 && tok->str() == ":")
initList = true;
}
}
// Update the variable ids..
// Parse each function..
static void setVarIdClassFunction(Token * const startToken,
const Token * const endToken,
const std::map<std::string, unsigned int> &varlist,
std::map<unsigned int, std::map<std::string,unsigned int> > *structMembers,
unsigned int *_varId)
{
for (Token *tok2 = startToken; tok2 && tok2 != endToken; tok2 = tok2->next()) {
if (tok2->varId() == 0 && tok2->previous()->str() != ".") {
const std::map<std::string,unsigned int>::const_iterator it = varlist.find(tok2->str());
if (it != varlist.end()) {
tok2->varId(it->second);
setVarIdStructMembers(&tok2, structMembers, _varId);
}
}
}
}
void Tokenizer::setVarId()
{
// Clear all variable ids
for (Token *tok = list.front(); tok; tok = tok->next())
tok->varId(0);
// Variable declarations can't start with "return" etc.
std::set<std::string> notstart;
notstart.insert("goto");
notstart.insert("NOT");
notstart.insert("return");
notstart.insert("sizeof");
if (!isC()) {
static const char *str[] = {"delete","friend","new","throw","using","virtual","explicit"};
notstart.insert(str, str+(sizeof(str)/sizeof(*str)));
}
// variable id
_varId = 0;
std::map<std::string, unsigned int> variableId;
std::map<unsigned int, std::map<std::string, unsigned int> > structMembers;
std::stack< std::map<std::string, unsigned int> > scopeInfo;
std::stack<bool> executableScope;
executableScope.push(false);
std::stack<unsigned int> scopestartvarid; // varid when scope starts
scopestartvarid.push(0);
for (Token *tok = list.front(); tok; tok = tok->next()) {
// scope info to handle shadow variables..
if (tok->str() == "(" &&
(Token::simpleMatch(tok->link(), ") {") || Token::Match(tok->link(), ") %type% {"))) {
scopeInfo.push(variableId);
} else if (tok->str() == "{") {
scopestartvarid.push(_varId);
if (Token::simpleMatch(tok->previous(), ")") || Token::Match(tok->tokAt(-2), ") %type%")) {
executableScope.push(true);
} else {
executableScope.push(executableScope.top());
scopeInfo.push(variableId);
}
} else if (tok->str() == "}") {
// Set variable ids in class declaration..
if (!isC() && !executableScope.top() && tok->link()) {
setVarIdClassDeclaration(tok->link(),
variableId,
scopestartvarid.top(),
&structMembers,
&_varId);
}
scopestartvarid.pop();
if (scopestartvarid.empty()) { // should be impossible
scopestartvarid.push(0);
}
if (scopeInfo.empty()) {
variableId.clear();
} else {
variableId.swap(scopeInfo.top());
scopeInfo.pop();
}
executableScope.pop();
if (executableScope.empty()) { // should not possibly happen
executableScope.push(false);
}
}
if (tok == list.front() || Token::Match(tok, "[;{}]") ||
(Token::Match(tok,"[(,]") && (!executableScope.top() || Token::simpleMatch(tok->link(), ") {"))) ||
(tok->isName() && tok->str().at(tok->str().length()-1U) == ':')) {
// No variable declarations in sizeof
if (Token::simpleMatch(tok->previous(), "sizeof (")) {
continue;
}
// locate the variable name..
const Token *tok2 = (tok->isName()) ? tok : tok->next();
// private: protected: public: etc
while (tok2 && tok2->str()[tok2->str().size() - 1U] == ':') {
tok2 = tok2->next();
}
if (!tok2)
break;
// Variable declaration can't start with "return", etc
if (notstart.find(tok2->str()) != notstart.end())
continue;
const bool decl = setVarIdParseDeclaration(&tok2, variableId, executableScope.top());
if (decl && Token::Match(tok2->previous(), "%type% [;[=,)]") && tok2->previous()->str() != "const") {
variableId[tok2->previous()->str()] = ++_varId;
tok = tok2->previous();
}
else if (decl && Token::Match(tok2->previous(), "%type% ( !!)") && Token::simpleMatch(tok2->link(), ") ;")) {
// In C++ , a variable can't be called operator+ or something like that.
if (isCPP() &&
tok2->previous()->str().size() >= 9 &&
tok2->previous()->str().compare(0, 8, "operator") == 0 &&
tok2->previous()->str()[8] != '_' &&
!std::isalnum(tok2->previous()->str()[8]))
continue;
const Token *tok3 = tok2->next();
if (!tok3->isStandardType() && !setVarIdParseDeclaration(&tok3,variableId,executableScope.top())) {
variableId[tok2->previous()->str()] = ++_varId;
tok = tok2->previous();
}
}
}
if (tok->isName()) {
// don't set variable id after a struct|enum|union
if (Token::Match(tok->previous(), "struct|enum|union"))
continue;
if (!isC()) {
if (tok->previous() && tok->previous()->str() == "::")
continue;
if (tok->next() && tok->next()->str() == "::")
continue;
}
const std::map<std::string, unsigned int>::const_iterator it = variableId.find(tok->str());
if (it != variableId.end()) {
tok->varId(it->second);
setVarIdStructMembers(&tok, &structMembers, &_varId);
}
} else if (Token::Match(tok, "::|. %var%")) {
// Don't set varid after a :: or . token
tok = tok->next();
} else if (tok->str() == ":" && Token::Match(tok->tokAt(-2), "class %type%")) {
do {
tok = tok->next();
} while (tok && (tok->isName() || tok->str() == ","));
if (!tok)
break;
tok = tok->previous();
}
}
// Clear the structMembers because it will be used when member functions
// are parsed. The old info is not bad, it is just redundant.
structMembers.clear();
// Member functions and variables in this source
std::list<Token *> allMemberFunctions;
std::list<Token *> allMemberVars;
{
for (Token *tok2 = list.front(); tok2; tok2 = tok2->next()) {
if (Token::Match(tok2, "%var% :: %var%")) {
if (tok2->strAt(3) == "(")
allMemberFunctions.push_back(tok2);
else if (tok2->tokAt(2)->varId() != 0)
allMemberVars.push_back(tok2);
}
}
}
// class members..
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "class|struct %var% {|:")) {
const std::string &classname(tok->next()->str());
// What member variables are there in this class?
std::map<std::string, unsigned int> varlist;
const Token* tokStart = Token::findsimplematch(tok, "{");
if (tokStart) {
for (const Token *tok2 = tokStart->next(); tok2 != tok->link(); tok2 = tok2->next()) {
// skip parentheses..
if (tok2->str() == "{")
tok2 = tok2->link();
else if (tok2->str() == "(")
tok2 = tok2->link();
// Found a member variable..
else if (tok2->varId() > 0)
varlist[tok2->str()] = tok2->varId();
}
}
// Are there any member variables in this class?
if (varlist.empty())
continue;
// Member variables
for (std::list<Token *>::iterator func = allMemberVars.begin(); func != allMemberVars.end(); ++func) {
if (!Token::simpleMatch(*func, classname.c_str()))
continue;
Token *tok2 = *func;
tok2 = tok2->tokAt(2);
tok2->varId(varlist[tok2->str()]);
}
// Set variable ids in member functions for this class..
const std::string funcpattern(classname + " :: %var% (");
for (std::list<Token *>::iterator func = allMemberFunctions.begin(); func != allMemberFunctions.end(); ++func) {
Token *tok2 = *func;
// Found a class function..
if (Token::Match(tok2, funcpattern.c_str())) {
// Goto the end parenthesis..
tok2 = tok2->linkAt(3);
if (!tok2)
break;
// If this is a function implementation.. add it to funclist
if (Token::Match(tok2, ") const|volatile| {")) {
while (tok2->str() != "{")
tok2 = tok2->next();
setVarIdClassFunction(tok2, tok2->link(), varlist, &structMembers, &_varId);
}
// constructor with initializer list
if (Token::Match(tok2, ") : %var% (")) {
const Token *tok3 = tok2;
while (Token::Match(tok3, ") [:,] %var% (")) {
tok3 = tok3->linkAt(3);
}
if (Token::simpleMatch(tok3, ") {")) {
setVarIdClassFunction(tok2, tok3->next()->link(), varlist, &structMembers, &_varId);
}
}
}
}
}
}
}
static bool linkBrackets(Tokenizer* tokenizer, std::stack<const Token*>& type, std::stack<Token*>& links, Token* token, char open, char close)
{
if (token->str()[0] == open) {
links.push(token);
type.push(token);
} else if (token->str()[0] == close) {
if (links.empty()) {
// Error, { and } don't match.
tokenizer->syntaxError(token, open);
return false;
}
if (type.top()->str()[0] != open) {
tokenizer->syntaxError(type.top(), type.top()->str()[0]);
return false;
}
type.pop();
Token::createMutualLinks(links.top(), token);
links.pop();
}
return(true);
}
bool Tokenizer::createLinks()
{
std::stack<const Token*> type;
std::stack<Token*> links1;
std::stack<Token*> links2;
std::stack<Token*> links3;
for (Token *token = list.front(); token; token = token->next()) {
if (token->link()) {
token->link(0);
}
bool validSyntax = linkBrackets(this, type, links1, token, '{', '}');
if (!validSyntax)
return false;
validSyntax = linkBrackets(this, type, links2, token, '(', ')');
if (!validSyntax)
return false;
validSyntax = linkBrackets(this, type, links3, token, '[', ']');
if (!validSyntax)
return false;
}
if (!links1.empty()) {
// Error, { and } don't match.
syntaxError(links1.top(), '{');
return false;
}
if (!links2.empty()) {
// Error, ( and ) don't match.
syntaxError(links2.top(), '(');
return false;
}
if (!links3.empty()) {
// Error, [ and ] don't match.
syntaxError(links3.top(), '[');
return false;
}
return true;
}
void Tokenizer::createLinks2()
{
if (isC())
return;
std::stack<Token*> type;
for (Token *token = list.front(); token; token = token->next()) {
if (token->link()) {
if (Token::Match(token, "{|[|("))
type.push(token);
else if (Token::Match(token, "}|]|)")) {
while (type.top()->str() == "<")
type.pop();
type.pop();
} else
token->link(0);
}
else if (token->str() == ";")
while (!type.empty() && type.top()->str() == "<")
type.pop();
else if (token->str() == "<" && token->previous() && token->previous()->isName() && !token->previous()->varId())
type.push(token);
else if (token->str() == ">" || token->str() == ">>") {
if (type.empty() || type.top()->str() != "<") // < and > don't match.
continue;
if (token->next() && !token->next()->isName() && !Token::Match(token->next(), ">|&|*|::|,|(|)"))
continue;
// Check type of open link
if (type.empty() || type.top()->str() != "<" || (token->str() == ">>" && type.size() < 2)) {
continue;
}
Token* top = type.top();
type.pop();
if (token->str() == ">>" && type.top()->str() != "<") {
type.push(top);
continue;
}
if (token->str() == ">>") { // C++11 right angle bracket
token->str(">");
token->insertToken(">");
}
Token::createMutualLinks(top, token);
}
}
}
bool Tokenizer::simplifySizeof()
{
// Locate variable declarations and calculate the size
std::map<unsigned int, std::string> sizeOfVar;
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->varId() != 0 && sizeOfVar.find(tok->varId()) == sizeOfVar.end()) {
const unsigned int varId = tok->varId();
if (Token::Match(tok->tokAt(-3), "[;{}(,] %type% * %var% [;,)]") ||
Token::Match(tok->tokAt(-4), "[;{}(,] const %type% * %var% [;),]") ||
Token::Match(tok->tokAt(-2), "[;{}(,] %type% %var% [;),]") ||
Token::Match(tok->tokAt(-3), "[;{}(,] const %type% %var% [;),]")) {
const unsigned int size = sizeOfType(tok->previous());
if (size == 0) {
continue;
}
sizeOfVar[varId] = MathLib::longToString(size);
}
else if (Token::Match(tok->previous(), "%type% %var% [ %num% ] [;=]") ||
Token::Match(tok->tokAt(-2), "%type% * %var% [ %num% ] [;=]")) {
const unsigned int size = sizeOfType(tok->previous());
if (size == 0)
continue;
sizeOfVar[varId] = MathLib::longToString(size * static_cast<unsigned long>(MathLib::toLongNumber(tok->strAt(2))));
}
else if (Token::Match(tok->previous(), "%type% %var% [ %num% ] [,)]") ||
Token::Match(tok->tokAt(-2), "%type% * %var% [ %num% ] [,)]")) {
Token tempTok(0);
tempTok.str("*");
sizeOfVar[varId] = MathLib::longToString(sizeOfType(&tempTok));
}
else if (Token::Match(tok->previous(), "%type% %var% [ ] = %str% ;")) {
const unsigned int size = sizeOfType(tok->tokAt(4));
if (size == 0)
continue;
sizeOfVar[varId] = MathLib::longToString(size);
}
}
}
bool ret = false;
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str() != "sizeof")
continue;
if (!tok->next())
break;
if (tok->strAt(1) == "sizeof")
continue;
if (Token::simpleMatch(tok->next(), ". . .")) {
tok->deleteNext(3);
}
// sizeof 'x'
if (tok->next()->type() == Token::eChar) {
tok->deleteThis();
std::ostringstream sz;
sz << sizeof 'x';
tok->str(sz.str());
ret = true;
continue;
}
// sizeof('x')
if (Token::Match(tok, "sizeof ( %any% )") && tok->strAt(2)[0] == '\'') {
tok->deleteNext();
tok->deleteThis();
tok->deleteNext();
std::ostringstream sz;
sz << sizeof 'x';
tok->str(sz.str());
ret = true;
continue;
}
// sizeof "text"
if (tok->next()->type() == Token::eString) {
tok->deleteThis();
std::ostringstream ostr;
ostr << (Token::getStrLength(tok) + 1);
tok->str(ostr.str());
ret = true;
continue;
}
// sizeof ("text")
if (Token::Match(tok->next(), "( %str% )")) {
tok->deleteNext();
tok->deleteThis();
tok->deleteNext();
std::ostringstream ostr;
ostr << (Token::getStrLength(tok) + 1);
tok->str(ostr.str());
ret = true;
continue;
}
// sizeof * (...) -> sizeof(*...)
if (Token::simpleMatch(tok->next(), "* (") && !Token::simpleMatch(tok->linkAt(2), ") .")) {
tok->deleteNext();
tok->next()->insertToken("*");
}
// sizeof a++ -> sizeof(a++)
if (Token::Match(tok->next(), "++|-- %var% !!.") || Token::Match(tok->next(), "%var% ++|--")) {
tok->insertToken("(");
tok->tokAt(3)->insertToken(")");
Token::createMutualLinks(tok->next(), tok->tokAt(4));
}
// sizeof 1 => sizeof ( 1 )
if (tok->next()->isNumber()) {
Token *tok2 = tok->next();
tok->insertToken("(");
tok2->insertToken(")");
Token::createMutualLinks(tok->next(), tok2->next());
}
// sizeof int -> sizeof( int )
else if (tok->next()->str() != "(") {
// Add parenthesis around the sizeof
int parlevel = 0;
for (Token *tempToken = tok->next(); tempToken; tempToken = tempToken->next()) {
if (tempToken->str() == "(")
++parlevel;
else if (tempToken->str() == ")")
--parlevel;
if (Token::Match(tempToken, "%var%")) {
while (tempToken && tempToken->next() && tempToken->next()->str() == "[") {
tempToken = tempToken->next()->link();
}
if (!tempToken || !tempToken->next()) {
break;
}
if (tempToken->next()->str() == ".") {
// We are checking a class or struct, search next varname
tempToken = tempToken->next();
continue;
} else if (tempToken->next()->type() == Token::eIncDecOp) {
// We have variable++ or variable--, there should be
// nothing after this
tempToken = tempToken->tokAt(2);
} else if (parlevel > 0 && Token::simpleMatch(tempToken->next(), ") .")) {
--parlevel;
tempToken = tempToken->tokAt(2);
continue;
}
// Ok, we should be clean. Add ) after tempToken
tok->insertToken("(");
tempToken->insertToken(")");
Token::createMutualLinks(tok->next(), tempToken->next());
break;
}
}
}
// sizeof(type *) => sizeof(*)
if (Token::Match(tok->next(), "( %type% * )")) {
tok->next()->deleteNext();
}
if (Token::simpleMatch(tok->next(), "( * )")) {
tok->str(MathLib::longToString(sizeOfType(tok->tokAt(2))));
tok->deleteNext(3);
ret = true;
}
// sizeof( a )
else if (Token::Match(tok->next(), "( %var% )") && tok->tokAt(2)->varId() != 0) {
if (sizeOfVar.find(tok->tokAt(2)->varId()) != sizeOfVar.end()) {
tok->deleteNext();
tok->deleteThis();
tok->deleteNext();
tok->str(sizeOfVar[tok->varId()]);
ret = true;
} else {
// don't try to replace size of variable if variable has
// similar name with type (#329)
}
}
else if (Token::Match(tok, "sizeof ( %type% )")) {
unsigned int size = sizeOfType(tok->tokAt(2));
if (size > 0) {
tok->str(MathLib::longToString(size));
tok->deleteNext(3);
ret = true;
}
}
else if (Token::Match(tok, "sizeof ( * %var% )") || Token::Match(tok, "sizeof ( %var% [ %num% ] )")) {
// Some default value..
std::size_t sz = 0;
unsigned int varid = tok->tokAt((tok->strAt(2) == "*") ? 3 : 2)->varId();
if (varid != 0) {
// Try to locate variable declaration..
const Token *decltok = Token::findmatch(list.front(), "%varid%", varid);
if (Token::Match(decltok->previous(), "%type% %var% [")) {
sz = sizeOfType(decltok->previous());
} else if (Token::Match(decltok->previous(), "* %var% [")) {
sz = sizeOfType(decltok->previous());
} else if (Token::Match(decltok->tokAt(-2), "%type% * %var%")) {
sz = sizeOfType(decltok->tokAt(-2));
}
} else if (tok->strAt(3) == "[" && tok->tokAt(2)->isStandardType()) {
sz = sizeOfType(tok->tokAt(2));
if (sz == 0)
continue;
sz *= static_cast<unsigned long>(MathLib::toLongNumber(tok->strAt(4)));
}
if (sz > 0) {
tok->str(MathLib::longToString(sz));
Token::eraseTokens(tok, tok->next()->link()->next());
ret = true;
}
}
}
return ret;
}
bool Tokenizer::simplifyTokenList()
{
// clear the _functionList so it can't contain dead pointers
deleteSymbolDatabase();
// simplify references
simplifyReference();
simplifyStd();
simplifyGoto();
simplifySizeof();
simplifyUndefinedSizeArray();
// Replace constants..
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "const %type% %var% = %num% ;")) {
unsigned int varId = tok->tokAt(2)->varId();
if (varId == 0) {
tok = tok->tokAt(5);
continue;
}
const std::string& num = tok->strAt(4);
int indent = 1;
for (Token *tok2 = tok->tokAt(6); tok2; tok2 = tok2->next()) {
if (tok2->str() == "{") {
++indent;
} else if (tok2->str() == "}") {
--indent;
if (indent == 0)
break;
}
// Compare constants, but don't touch members of other structures
else if (tok2->varId() == varId) {
tok2->str(num);
}
}
}
}
simplifyCasts();
// Simplify simple calculations..
simplifyCalculations();
// Replace "*(str + num)" => "str[num]"
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (!Token::Match(tok, "%var%") && !tok->isNumber()
&& !Token::Match(tok, "]|)")
&& (Token::Match(tok->next(), "* ( %var% + %num% )") ||
Token::Match(tok->next(), "* ( %var% + %var% )"))) {
// remove '* ('
tok->deleteNext(2);
tok = tok->tokAt(2);
// '+'->'['
tok->str("[");
tok = tok->tokAt(2);
tok->str("]");
Token::createMutualLinks(tok->tokAt(-2), tok);
}
}
// Replace "&str[num]" => "(str + num)"
std::set<unsigned int> pod;
for (const Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->isStandardType()) {
tok = tok->next();
while (tok && (tok->str() == "*" || tok->isName())) {
if (tok->varId() > 0) {
pod.insert(tok->varId());
break;
}
tok = tok->next();
}
if (!tok)
break;
}
}
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (!Token::Match(tok, "%var%")
&& !Token::Match(tok, "%num%")
&& !Token::Match(tok, "]|)")
&& (Token::Match(tok->next(), "& %var% [ %num% ]") ||
Token::Match(tok->next(), "& %var% [ %var% ]"))) {
tok = tok->next();
if (tok->next()->varId()) {
if (pod.find(tok->next()->varId()) == pod.end()) {
tok = tok->tokAt(5);
continue;
}
}
// '&' => '('
tok->str("(");
tok = tok->next();
// '[' => '+'
tok->deleteNext();
tok->insertToken("+");
tok = tok->tokAt(3);
//remove ']'
tok->str(")");
Token::createMutualLinks(tok->tokAt(-4), tok);
}
}
removeRedundantAssignment();
simplifyRealloc();
// Change initialisation of variable to assignment
simplifyInitVar();
// Simplify variable declarations
simplifyVarDecl(false);
elseif();
simplifyErrNoInWhile();
simplifyIfAssign();
simplifyRedundantParenthesis();
simplifyIfNot();
simplifyIfNotNull();
simplifyIfSameInnerCondition();
simplifyComparisonOrder();
simplifyNestedStrcat();
simplifyWhile0();
simplifyFuncInWhile();
simplifyIfAssign(); // could be affected by simplifyIfNot
bool modified = true;
while (modified) {
modified = false;
modified |= simplifyConditions();
modified |= simplifyFunctionReturn();
modified |= simplifyKnownVariables();
modified |= removeRedundantConditions();
modified |= simplifyRedundantParenthesis();
modified |= simplifyQuestionMark();
modified |= simplifyCalculations();
}
simplifyConditionOperator();
// replace strlen(str)
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "strlen ( %str% )")) {
std::ostringstream ostr;
ostr << Token::getStrLength(tok->tokAt(2));
tok->str(ostr.str());
tok->deleteNext(3);
}
}
// simplify redundant for
removeRedundantFor();
// Remove redundant parentheses in return..
for (Token *tok = list.front(); tok; tok = tok->next()) {
while (Token::simpleMatch(tok, "return (")) {
Token *tok2 = tok->next()->link();
if (Token::simpleMatch(tok2, ") ;")) {
tok->deleteNext();
tok2->deleteThis();
} else {
break;
}
}
}
simplifyReturnStrncat();
removeRedundantAssignment();
simplifyComma();
removeRedundantSemicolons();
simplifyFlowControl();
simplifyRedundantConsecutiveBraces();
simplifyEmptyNamespaces();
if (!validate())
return false;
list.front()->assignProgressValues();
// Create symbol database and then remove const keywords
createSymbolDatabase();
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::simpleMatch(tok, "* const"))
tok->deleteNext();
}
if (_settings->debug) {
list.front()->printOut(0, list.getFiles());
if (_settings->_verbose)
_symbolDatabase->printOut("Symbol database");
}
if (_settings->debugwarnings) {
printUnknownTypes();
}
return true;
}
//---------------------------------------------------------------------------
void Tokenizer::removeMacrosInGlobalScope()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str() == "(") {
tok = tok->link();
if (Token::Match(tok, ") %type% {") &&
!Token::Match(tok->next(), "const|namespace|class|struct|union"))
tok->deleteNext();
}
if (Token::Match(tok, "[;{}] %type%") && tok->next()->isUpperCaseName()) {
const Token *tok2 = tok->tokAt(2);
if (tok2 && tok2->str() == "(") {
unsigned int par = 0;
for (; tok2; tok2 = tok2->next()) {
if (tok2->str() == "(")
++par;
else if (tok2->str() == ")") {
if (par <= 1)
break;
--par;
}
}
if (tok2 && tok2->str() == ")")
tok2 = tok2->next();
}
// remove unknown macros before namespace|class|struct|union
if (Token::Match(tok2, "namespace|class|struct|union")) {
// is there a "{" for?
const Token *tok3 = tok2;
while (tok3 && !Token::Match(tok3,"[;{}()]"))
tok3 = tok3->next();
if (tok3 && tok3->str() == "{")
Token::eraseTokens(tok, tok2);
continue;
}
// remove unknown macros before foo::foo(
if (Token::Match(tok2, "%type% :: %type%")) {
const Token *tok3 = tok2;
while (Token::Match(tok3, "%type% :: %type% ::"))
tok3 = tok3->tokAt(2);
if (Token::Match(tok3, "%type% :: %type% (") && tok3->str() == tok3->strAt(2))
Token::eraseTokens(tok, tok2);
continue;
}
}
if (tok->str() == "{")
tok = tok->link();
}
}
//---------------------------------------------------------------------------
void Tokenizer::removeRedundantAssignment()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str() == "{")
tok = tok->link();
if (Token::Match(tok, ") const| {")) {
// parse in this function..
std::set<unsigned int> localvars;
if (tok->next()->str() == "const")
tok = tok->next();
const Token * const end = tok->next()->link();
for (Token *tok2 = tok->next(); tok2 && tok2 != end; tok2 = tok2->next()) {
// skip local class or struct
if (Token::Match(tok2, "class|struct %type% {|:")) {
// skip to '{'
while (tok2 && tok2->str() != "{")
tok2 = tok2->next();
if (tok2)
tok2 = tok2->link(); // skip local class or struct
else
return;
} else if (Token::Match(tok2, "[;{}] %type% * %var% ;") && tok2->next()->str() != "return") {
tok2 = tok2->tokAt(3);
localvars.insert(tok2->varId());
} else if (Token::Match(tok2, "[;{}] %type% %var% ;") && tok2->next()->isStandardType()) {
tok2 = tok2->tokAt(2);
localvars.insert(tok2->varId());
} else if (tok2->varId() &&
!Token::Match(tok2->previous(), "[;{}] %var% = %var% ;") &&
!Token::Match(tok2->previous(), "[;{}] %var% = %num% ;") &&
!(Token::Match(tok2->previous(), "[;{}] %var% = %any% ;") && tok2->strAt(2)[0] == '\'')) {
localvars.erase(tok2->varId());
}
}
localvars.erase(0);
if (!localvars.empty()) {
for (Token *tok2 = tok->next(); tok2 && tok2 != end;) {
if (Token::Match(tok2, "[;{}] %type% %var% ;") && localvars.find(tok2->tokAt(2)->varId()) != localvars.end()) {
tok2->deleteNext(3);
} else if ((Token::Match(tok2, "[;{}] %type% * %var% ;") &&
localvars.find(tok2->tokAt(3)->varId()) != localvars.end()) ||
(Token::Match(tok2, "[;{}] %var% = %any% ;") &&
localvars.find(tok2->next()->varId()) != localvars.end())) {
tok2->deleteNext(4);
} else
tok2 = tok2->next();
}
}
}
}
}
void Tokenizer::simplifyRealloc()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str() == "(" || tok->str() == "[" ||
(tok->str() == "{" && tok->previous() && tok->previous()->str() == "="))
tok = tok->link();
else if (Token::Match(tok, "[;{}] %var% = realloc (")) {
tok = tok->tokAt(3);
if (Token::simpleMatch(tok->next(), "( 0 ,")) {
//no "x = realloc(0,);"
if (!Token::simpleMatch(tok->next()->link(), ") ;") || tok->next()->link()->previous() == tok->tokAt(3))
continue;
// delete "0 ,"
tok->next()->deleteNext(2);
// Change function name "realloc" to "malloc"
tok->str("malloc");
tok = tok->next()->link();
} else {
Token *tok2 = tok->next()->link()->tokAt(-2);
//no "x = realloc(,0);"
if (!Token::simpleMatch(tok2, ", 0 ) ;") || tok2 == tok->tokAt(2))
continue;
//remove ", 0"
tok2 = tok2->previous();
tok2->deleteNext(2);
//change "realloc" to "free"
tok->str("free");
//insert "0" after "var ="
tok = tok->previous();
tok->insertToken("0");
//move "var = 0" between "free(...)" and ";"
tok2 = tok2->next();
Token::move(tok->previous(), tok->next(), tok2);
//add missing ";" after "free(...)"
tok2->insertToken(";");
//goto before last ";" and continue
tok = tok->next();
}
}
}
}
void Tokenizer::simplifyEmptyNamespaces()
{
if (isC())
return;
bool goback = false;
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (goback) {
tok = tok->previous();
goback = false;
}
if (tok->str() == "(" || tok->str() == "[" || tok->str() == "{") {
tok = tok->link();
continue;
}
if (!Token::Match(tok, "namespace %var% {"))
continue;
if (tok->strAt(3) == "}") {
tok->deleteNext(3); // remove '%var% { }'
if (!tok->previous()) {
// remove 'namespace' or replace it with ';' if isolated
tok->deleteThis();
goback = true;
} else { // '%any% namespace %any%'
tok = tok->pre