pass language to TokenList constructor

lib/clangimport.cpp

@@ -677,8 +677,7 @@ void clangimport::AstNode::setValueType(Token *tok)
             // TODO
             continue;
 
-        TokenList decl(nullptr);
-        decl.setLang(tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
+        TokenList decl(nullptr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
         addTypeTokens(decl, type, tok->scope());
         if (!decl.front())
             break;

lib/cppcheck.cpp

@@ -715,7 +715,7 @@ unsigned int CppCheck::checkClang(const FileWithDetails &file)
     }
 
     try {
-        TokenList tokenlist{&mSettings};
+        TokenList tokenlist{&mSettings, file.lang()};
         tokenlist.appendFileIfNew(file.spath());
         Tokenizer tokenizer(std::move(tokenlist), mSettings, mErrorLogger);
         std::istringstream ast(output2);
@@ -910,9 +910,8 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
 
             if (mUnusedFunctionsCheck && (mSettings.useSingleJob() || analyzerInformation)) {
                 std::size_t hash = 0;
-                TokenList tokenlist{&mSettings};
-                // enforce the language since markup files are special and do not adhere to the enforced language
-                tokenlist.setLang(Standards::Language::C, true);
+                // markup files are special and do not adhere to the enforced language
+                TokenList tokenlist{&mSettings, Standards::Language::C};
                 if (fileStream) {
                     std::vector<std::string> files;
                     simplecpp::TokenList tokens(*fileStream, files, file.spath());
@@ -1056,10 +1055,9 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
                 if (startsWith(dir.str,"#define ") || startsWith(dir.str,"#include "))
                     code += "#line " + std::to_string(dir.linenr) + " \"" + dir.file + "\"\n" + dir.str + '\n';
             }
-            TokenList tokenlist(&mSettings);
+            TokenList tokenlist(&mSettings, file.lang());
             std::istringstream istr2(code);
-            // TODO: asserts when file has unknown extension
-            tokenlist.createTokens(istr2, Path::identify(*files.begin(), false)); // TODO: check result?
+            tokenlist.createTokens(istr2); // TODO: check result?
             executeRules("define", tokenlist);
         }
 #endif
@@ -1135,7 +1133,7 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
             }
 
             try {
-                TokenList tokenlist{&mSettings};
+                TokenList tokenlist{&mSettings, file.lang()};
 
                 // Create tokens, skip rest of iteration if failed
                 Timer::run("Tokenizer::createTokens", mSettings.showtime, &s_timerResults, [&]() {

lib/importproject.cpp

@@ -577,9 +577,9 @@ namespace {
 
             // TODO: improve evaluation
             const Settings s;
-            TokenList tokenlist(&s);
+            TokenList tokenlist(&s, Standards::Language::C);
             std::istringstream istr(c);
-            tokenlist.createTokens(istr, Standards::Language::C); // TODO: check result
+            tokenlist.createTokens(istr); // TODO: check result
             // TODO: put in a helper
             // generate links
             {

lib/library.cpp

@@ -175,10 +175,10 @@ static std::vector<std::string> getnames(const char *names)
     return ret;
 }
 
-static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList)
+static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList)
 {
     std::istringstream istr(valid + ',');
-    tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
+    tokenList.createTokens(istr); // TODO: check result?
     for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok,"- %num%")) {
             tok->str("-" + tok->strAt(1));
@@ -1062,8 +1062,8 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint
         return true;
     if (ac->valid.find('.') != std::string::npos)
         return isFloatArgValid(ftok, argnr, static_cast<double>(argvalue));
-    TokenList tokenList(nullptr);
-    gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
+    TokenList tokenList(nullptr, ftok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
+    gettokenlistfromvalid(ac->valid, tokenList);
     for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok))
             return true;
@@ -1082,8 +1082,8 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con
     const ArgumentChecks *ac = getarg(ftok, argnr);
     if (!ac || ac->valid.empty())
         return true;
-    TokenList tokenList(nullptr);
-    gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
+    TokenList tokenList(nullptr, ftok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
+    gettokenlistfromvalid(ac->valid, tokenList);
     for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok) && argvalue <= MathLib::toDoubleNumber(tok->tokAt(2)))
             return true;

lib/programmemory.cpp

@@ -1750,11 +1750,11 @@ static std::shared_ptr<Token> createTokenFromExpression(const std::string& retur
                                                         bool cpp,
                                                         std::unordered_map<nonneg int, const Token*>& lookupVarId)
 {
-    std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(&settings);
+    std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(&settings, cpp ? Standards::Language::CPP : Standards::Language::C);
     {
         const std::string code = "return " + returnValue + ";";
         std::istringstream istr(code);
-        if (!tokenList->createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
+        if (!tokenList->createTokens(istr))
             return nullptr;
     }
 

lib/symboldatabase.cpp

@@ -1808,8 +1808,7 @@ void SymbolDatabase::setArrayDimensionsUsingValueFlow()
 
                 // In template arguments, there might not be AST
                 // Determine size by using the "raw tokens"
-                TokenList tokenList(&mSettings);
-                tokenList.setLang(dimension.tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
+                TokenList tokenList(&mSettings, dimension.tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
                 tokenList.addtoken(";", 0, 0, 0, false);
                 bool fail = false;
                 for (const Token *tok = dimension.tok; tok && !Token::Match(tok, "[,>]"); tok = tok->next()) {
@@ -7293,7 +7292,7 @@ static const Token* parsedecl(const Token* type,
         else if (Token::simpleMatch(type, "volatile"))
             valuetype->volatileness |= (1 << (valuetype->pointer - pointer0));
         else if (settings.clang && type->str().size() > 2 && type->str().find("::") < type->str().find('<')) {
-            TokenList typeTokens(&settings);
+            TokenList typeTokens(&settings, type->isCpp() ? Standards::Language::CPP : Standards::Language::C);
             std::string::size_type pos1 = 0;
             do {
                 const std::string::size_type pos2 = type->str().find("::", pos1);
@@ -7716,9 +7715,9 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                 const std::string& typestr(mSettings.library.returnValueType(tok->previous()));
                 if (!typestr.empty()) {
                     ValueType valuetype;
-                    TokenList tokenList(&mSettings);
+                    TokenList tokenList(&mSettings, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
                     std::istringstream istr(typestr+";");
-                    tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
+                    tokenList.createTokens(istr); // TODO: check result?
                     tokenList.simplifyStdType();
                     if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings)) {
                         valuetype.originalTypeName = typestr;
@@ -7806,9 +7805,9 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                     }
                     continue;
                 }
-                TokenList tokenList(&mSettings);
+                TokenList tokenList(&mSettings, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
                 std::istringstream istr(typestr+";");
-                if (tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C)) {
+                if (tokenList.createTokens(istr)) {
                     ValueType vt;
                     tokenList.simplifyPlatformTypes();
                     tokenList.simplifyStdType();

lib/tokenize.cpp

@@ -10870,7 +10870,7 @@ bool Tokenizer::isPacked(const Token * bodyStart) const
 
 void Tokenizer::getErrorMessages(ErrorLogger& errorLogger, const Settings& settings)
 {
-    TokenList tokenlist{&settings};
+    TokenList tokenlist{&settings, Standards::Language::C};
     Tokenizer tokenizer(std::move(tokenlist), settings, errorLogger);
     tokenizer.invalidConstFunctionTypeError(nullptr);
     // checkLibraryNoReturn

lib/tokenlist.cpp

@@ -60,13 +60,12 @@
 static constexpr int AST_MAX_DEPTH = 150;
 
 
-TokenList::TokenList(const Settings* settings)
+TokenList::TokenList(const Settings* settings, Standards::Language lang)
     : mTokensFrontBack(new TokensFrontBack)
     , mSettings(settings)
 {
-    if (mSettings && (mSettings->enforcedLang != Standards::Language::None)) {
-        mLang = mSettings->enforcedLang;
-    }
+    assert(lang != Standards::Language::None);
+    mLang = lang;
 }
 
 TokenList::~TokenList()
@@ -346,15 +345,8 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
 
 //---------------------------------------------------------------------------
 
-bool TokenList::createTokens(std::istream &code, Standards::Language lang)
+bool TokenList::createTokens(std::istream &code)
 {
-    ASSERT_LANG(lang != Standards::Language::None);
-    if (mLang == Standards::Language::None) {
-        mLang = lang;
-    } else {
-        ASSERT_LANG(lang == mLang);
-    }
-
     return createTokensInternal(code, mFiles.empty() ? "" : *mFiles.cbegin());
 }
 
@@ -2288,17 +2280,6 @@ bool TokenList::isCPP() const
     return mLang == Standards::Language::CPP;
 }
 
-void TokenList::setLang(Standards::Language lang, bool force)
-{
-    ASSERT_LANG(lang != Standards::Language::None);
-    if (!force)
-    {
-        ASSERT_LANG(mLang == Standards::Language::None);
-    }
-
-    mLang = lang;
-}
-
 const Token * TokenList::isFunctionHead(const Token *tok, const std::string &endsWith)
 {
     if (!tok)

lib/tokenlist.h

@@ -51,7 +51,7 @@ struct TokensFrontBack {
 class CPPCHECKLIB TokenList {
 public:
     // TODO: pass settings as reference
-    explicit TokenList(const Settings* settings);
+    explicit TokenList(const Settings* settings, Standards::Language lang);
     ~TokenList();
 
     TokenList(const TokenList &) = delete;
@@ -68,9 +68,6 @@ class CPPCHECKLIB TokenList {
     /** @return true if the code is C++ */
     bool isCPP() const;
 
-    // TODO: get rid of this
-    void setLang(Standards::Language lang, bool force = false);
-
     /**
      * Delete all tokens in given token list
      * @param tok token list to delete
@@ -103,9 +100,8 @@ class CPPCHECKLIB TokenList {
      * - UTF in the code are not handled.
      * - comments are not handled.
      * @param code input stream for code
-     * @param lang the language of the code
      */
-    bool createTokens(std::istream &code, Standards::Language lang);
+    bool createTokens(std::istream &code);
 
     void createTokens(simplecpp::TokenList&& tokenList);
 

lib/valueflow.cpp

@@ -1943,9 +1943,9 @@ static bool isNotEqual(std::pair<const Token*, const Token*> x, std::pair<const
 }
 static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y, bool cpp)
 {
-    TokenList tokenList(nullptr);
+    TokenList tokenList(nullptr, cpp ? Standards::Language::CPP : Standards::Language::C);
     std::istringstream istr(y);
-    tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
+    tokenList.createTokens(istr); // TODO: check result?
     return isNotEqual(x, std::make_pair(tokenList.front(), tokenList.back()));
 }
 static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y, bool cpp)
@@ -7051,9 +7051,9 @@ static bool getMinMaxValues(const std::string& typestr,
                             MathLib::bigint& minvalue,
                             MathLib::bigint& maxvalue)
 {
-    TokenList typeTokens(&settings);
+    TokenList typeTokens(&settings, cpp ? Standards::Language::CPP : Standards::Language::C);
     std::istringstream istr(typestr + ";");
-    if (!typeTokens.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
+    if (!typeTokens.createTokens(istr))
         return false;
     typeTokens.simplifyPlatformTypes();
     typeTokens.simplifyStdType();

test/helpers.h

@@ -45,21 +45,16 @@ namespace tinyxml2 {
 class SimpleTokenizer : public Tokenizer {
 public:
     explicit SimpleTokenizer(ErrorLogger& errorlogger, bool cpp = true)
-        : Tokenizer{TokenList{&s_settings}, s_settings, errorlogger}
-    {
-        list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true);
-    }
+        : Tokenizer{TokenList{&s_settings, cpp ? Standards::Language::CPP : Standards::Language::C}, s_settings, errorlogger}
+    {}
 
     SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, bool cpp = true)
-        : Tokenizer{TokenList{&settings}, settings, errorlogger}
-    {
-        list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true);
-    }
+        : Tokenizer{TokenList{&settings, cpp ? Standards::Language::CPP : Standards::Language::C}, settings, errorlogger}
+    {}
 
     SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const std::string& filename)
-        : Tokenizer{TokenList{&settings}, settings, errorlogger}
+        : Tokenizer{TokenList{&settings, Path::identify(filename, false)}, settings, errorlogger}
     {
-        list.setLang(Path::identify(filename, false));
         list.appendFileIfNew(filename);
     }
 
@@ -89,7 +84,7 @@ class SimpleTokenizer : public Tokenizer {
         if (list.front())
             throw std::runtime_error("token list is not empty");
         list.appendFileIfNew(filename);
-        if (!list.createTokens(istr, Path::identify(filename, false)))
+        if (!list.createTokens(istr))
             return false;
 
         return simplifyTokens1("");
@@ -104,9 +99,10 @@ class SimpleTokenList
 public:
     template<size_t size>
     explicit SimpleTokenList(const char (&code)[size], Standards::Language lang = Standards::Language::CPP)
+        : list{&settings, lang}
     {
         std::istringstream iss(code);
-        if (!list.createTokens(iss, lang))
+        if (!list.createTokens(iss))
             throw std::runtime_error("creating tokens failed");
     }
 
@@ -120,7 +116,7 @@ class SimpleTokenList
 
 private:
     const Settings settings;
-    TokenList list{&settings};
+    TokenList list;
 };
 
 
@@ -238,14 +234,14 @@ class SimpleTokenizer2 : public Tokenizer {
 public:
     template<size_t size>
     SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char (&code)[size], const std::string& file0)
-        : Tokenizer{TokenList{&settings}, settings, errorlogger}
+        : Tokenizer{TokenList{&settings, Path::identify(file0, false)}, settings, errorlogger}
     {
         preprocess(code, mFiles, file0, *this, errorlogger);
     }
 
     // TODO: get rid of this
     SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char code[], const std::string& file0)
-        : Tokenizer{TokenList{&settings}, settings, errorlogger}
+        : Tokenizer{TokenList{&settings, Path::identify(file0, false)}, settings, errorlogger}
     {
         preprocess(code, mFiles, file0, *this, errorlogger);
     }
@@ -263,7 +259,7 @@ struct TokenListHelper
         if (tokenlist.front())
             throw std::runtime_error("token list is not empty");
         tokenlist.appendFileIfNew(file);
-        return tokenlist.createTokens(istr, Path::identify(file, false));
+        return tokenlist.createTokens(istr);
     }
 };
 

test/testclangimport.cpp

@@ -140,7 +140,7 @@ class TestClangImport : public TestFixture {
 
     std::string parse(const char clang[]) {
         const Settings settings = settingsBuilder().clang().build();
-        TokenList tokenlist{&settings};
+        TokenList tokenlist{&settings, Standards::Language::CPP};
         Tokenizer tokenizer(std::move(tokenlist), settings, *this);
         std::istringstream istr(clang);
         clangimport::parseClangAstDump(tokenizer, istr);
@@ -1061,7 +1061,7 @@ class TestClangImport : public TestFixture {
 
 #define GET_SYMBOL_DB(AST) \
     const Settings settings = settingsBuilder().clang().platform(Platform::Type::Unix64).build(); \
-    TokenList tokenlist{&settings}; \
+    TokenList tokenlist{&settings, Standards::Language::CPP}; \
     Tokenizer tokenizer(std::move(tokenlist), settings, *this); \
     { \
         std::istringstream istr(AST); \