From 59ced7423ef68756e5ad45f09f2a2f154972ab19 Mon Sep 17 00:00:00 2001 From: firewave Date: Mon, 2 Sep 2024 18:38:12 +0200 Subject: [PATCH 1/5] bumped simplecpp to 1.1.4 --- externals/simplecpp/simplecpp.cpp | 70 ++++++++++++++++++++++++++++--- externals/simplecpp/simplecpp.h | 11 +++-- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/externals/simplecpp/simplecpp.cpp b/externals/simplecpp/simplecpp.cpp index 47bb78f8cf1..3cce780efca 100755 --- a/externals/simplecpp/simplecpp.cpp +++ b/externals/simplecpp/simplecpp.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -377,6 +378,42 @@ class StdIStream : public simplecpp::TokenList::Stream { std::istream &istr; }; +class StdCharBufStream : public simplecpp::TokenList::Stream { +public: + // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members + StdCharBufStream(const unsigned char* str, std::size_t size) + : str(str) + , size(size) + , pos(0) + , lastStatus(0) + { + init(); + } + + virtual int get() OVERRIDE { + if (pos >= size) + return lastStatus = EOF; + return str[pos++]; + } + virtual int peek() OVERRIDE { + if (pos >= size) + return lastStatus = EOF; + return str[pos]; + } + virtual void unget() OVERRIDE { + --pos; + } + virtual bool good() OVERRIDE { + return lastStatus != EOF; + } + +private: + const unsigned char *str; + const std::size_t size; + std::size_t pos; + int lastStatus; +}; + class FileStream : public simplecpp::TokenList::Stream { public: // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members @@ -442,6 +479,20 @@ simplecpp::TokenList::TokenList(std::istream &istr, std::vector &fi readfile(stream,filename,outputList); } +simplecpp::TokenList::TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList) + : frontToken(nullptr), backToken(nullptr), files(filenames) +{ + StdCharBufStream stream(data, size); + readfile(stream,filename,outputList); +} + +simplecpp::TokenList::TokenList(const char* data, std::size_t size, std::vector &filenames, const std::string &filename, OutputList *outputList) + : frontToken(nullptr), backToken(nullptr), files(filenames) +{ + StdCharBufStream stream(reinterpret_cast(data), size); + readfile(stream,filename,outputList); +} + simplecpp::TokenList::TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList) : frontToken(nullptr), backToken(nullptr), files(filenames) { @@ -1447,8 +1498,7 @@ namespace simplecpp { Macro(const std::string &name, const std::string &value, std::vector &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(false) { const std::string def(name + ' ' + value); - std::istringstream istr(def); - StdIStream stream(istr); + StdCharBufStream stream(reinterpret_cast(def.data()), def.size()); tokenListDefine.readfile(stream); if (!parseDefine(tokenListDefine.cfront())) throw std::runtime_error("bad macro syntax. macroname=" + name + " value=" + value); @@ -3315,8 +3365,17 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, dummy))); } else { std_def = simplecpp::getCppStdString(dui.std); - if (!std_def.empty()) - macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, dummy))); + if (std_def.empty()) { + if (outputList) { + simplecpp::Output err(files); + err.type = Output::DUI_ERROR; + err.msg = "unknown standard specified: '" + dui.std + "'"; + outputList->push_back(err); + } + output.clear(); + return; + } + macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, dummy))); } } @@ -3463,7 +3522,8 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL std::ifstream f; header2 = openHeader(f, dui, rawtok->location.file(), header, systemheader); if (f.is_open()) { - TokenList * const tokens = new TokenList(f, files, header2, outputList); + f.close(); + TokenList * const tokens = new TokenList(header2, files, outputList); if (dui.removeComments) tokens->removeComments(); filedata[header2] = tokens; diff --git a/externals/simplecpp/simplecpp.h b/externals/simplecpp/simplecpp.h index c72e4c655cf..88a14014567 100755 --- a/externals/simplecpp/simplecpp.h +++ b/externals/simplecpp/simplecpp.h @@ -1,4 +1,4 @@ -/* +/* -*- C++ -*- * simplecpp - A simple and high-fidelity C/C++ preprocessor library * Copyright (C) 2016-2023 simplecpp team */ @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include #include @@ -181,7 +181,8 @@ namespace simplecpp { PORTABILITY_BACKSLASH, UNHANDLED_CHAR_ERROR, EXPLICIT_INCLUDE_NOT_FOUND, - FILE_NOT_FOUND + FILE_NOT_FOUND, + DUI_ERROR } type; explicit Output(const std::vector& files, Type type, const std::string& msg) : type(type), location(files), msg(msg) {} Location location; @@ -198,6 +199,10 @@ namespace simplecpp { explicit TokenList(std::vector &filenames); /** generates a token list from the given std::istream parameter */ TokenList(std::istream &istr, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); + /** generates a token list from the given buffer */ + TokenList(const unsigned char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); + /** generates a token list from the given buffer */ + TokenList(const char* data, std::size_t size, std::vector &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr); /** generates a token list from the given filename parameter */ TokenList(const std::string &filename, std::vector &filenames, OutputList *outputList = nullptr); TokenList(const TokenList &other); From b738f02bfcba2bd210e5e40de09ad43faef96e73 Mon Sep 17 00:00:00 2001 From: firewave Date: Mon, 2 Sep 2024 19:46:58 +0200 Subject: [PATCH 2/5] preprocessor.cpp: handle `simplecpp::Output::DUI_ERROR` --- lib/preprocessor.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/preprocessor.cpp b/lib/preprocessor.cpp index eeb96362898..8bd20afb784 100644 --- a/lib/preprocessor.cpp +++ b/lib/preprocessor.cpp @@ -735,6 +735,7 @@ bool Preprocessor::hasErrors(const simplecpp::Output &output) case simplecpp::Output::UNHANDLED_CHAR_ERROR: case simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND: case simplecpp::Output::FILE_NOT_FOUND: + case simplecpp::Output::DUI_ERROR: return true; case simplecpp::Output::WARNING: case simplecpp::Output::MISSING_HEADER: @@ -875,6 +876,7 @@ void Preprocessor::reportOutput(const simplecpp::OutputList &outputList, bool sh break; case simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND: case simplecpp::Output::FILE_NOT_FOUND: + case simplecpp::Output::DUI_ERROR: error(emptyString, 0, out.msg); break; } From 45071ed2ff774a14e7a7183713c32d5b1da7d127 Mon Sep 17 00:00:00 2001 From: firewave Date: Mon, 2 Sep 2024 19:55:53 +0200 Subject: [PATCH 3/5] use more verbose shell output for addon tests in CI --- .github/workflows/CI-unixish.yml | 1 + .github/workflows/CI-windows.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml index 5649d3362f0..f150a853ef2 100644 --- a/.github/workflows/CI-unixish.yml +++ b/.github/workflows/CI-unixish.yml @@ -462,6 +462,7 @@ jobs: # TODO: move to scriptcheck.yml so these are tested with all Python versions? - name: Test addons run: | + set -x ./cppcheck --error-exitcode=1 --inline-suppr --addon=threadsafety addons/test/threadsafety ./cppcheck --error-exitcode=1 --inline-suppr --addon=threadsafety --std=c++03 addons/test/threadsafety ./cppcheck --error-exitcode=1 --inline-suppr --addon=misra addons/test/misra/crash*.c diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml index 8deb9b64c43..2d5f58ff7f5 100644 --- a/.github/workflows/CI-windows.yml +++ b/.github/workflows/CI-windows.yml @@ -194,6 +194,7 @@ jobs: - name: Test addons if: matrix.config == 'release' run: | + echo on .\cppcheck --addon=threadsafety addons\test\threadsafety || exit /b !errorlevel! .\cppcheck --addon=threadsafety --std=c++03 addons\test\threadsafety || exit /b !errorlevel! .\cppcheck --addon=misra --enable=style --inline-suppr --enable=information --error-exitcode=1 addons\test\misra\misra-ctu-*-test.c || exit /b !errorlevel! From 5989b81db0d87c33e084aae50890228906a81098 Mon Sep 17 00:00:00 2001 From: firewave Date: Mon, 9 Sep 2024 15:12:42 +0200 Subject: [PATCH 4/5] bumped simplecpp to 1.1.5 --- externals/simplecpp/simplecpp.cpp | 131 +++++++++++++++++++++--------- externals/simplecpp/simplecpp.h | 13 +++ 2 files changed, 105 insertions(+), 39 deletions(-) diff --git a/externals/simplecpp/simplecpp.cpp b/externals/simplecpp/simplecpp.cpp index 3cce780efca..d2fa6ee3082 100755 --- a/externals/simplecpp/simplecpp.cpp +++ b/externals/simplecpp/simplecpp.cpp @@ -3360,12 +3360,14 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL macros.insert(std::make_pair("__TIME__", Macro("__TIME__", getTimeDefine(<ime), dummy))); if (!dui.std.empty()) { - std::string std_def = simplecpp::getCStdString(dui.std); - if (!std_def.empty()) { - macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, dummy))); + const cstd_t c_std = simplecpp::getCStd(dui.std); + if (c_std != CUnknown) { + const std::string std_def = simplecpp::getCStdString(c_std); + if (!std_def.empty()) + macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, dummy))); } else { - std_def = simplecpp::getCppStdString(dui.std); - if (std_def.empty()) { + const cppstd_t cpp_std = simplecpp::getCppStd(dui.std); + if (cpp_std == CPPUnknown) { if (outputList) { simplecpp::Output err(files); err.type = Output::DUI_ERROR; @@ -3375,7 +3377,9 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL output.clear(); return; } - macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, dummy))); + const std::string std_def = simplecpp::getCppStdString(cpp_std); + if (!std_def.empty()) + macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, dummy))); } } @@ -3786,56 +3790,105 @@ void simplecpp::cleanup(std::map &filedata) filedata.clear(); } -std::string simplecpp::getCStdString(const std::string &std) +simplecpp::cstd_t simplecpp::getCStd(const std::string &std) { - if (std == "c90" || std == "c89" || std == "iso9899:1990" || std == "iso9899:199409" || std == "gnu90" || std == "gnu89") { - // __STDC_VERSION__ is not set for C90 although the macro was added in the 1994 amendments - return ""; - } + if (std == "c90" || std == "c89" || std == "iso9899:1990" || std == "iso9899:199409" || std == "gnu90" || std == "gnu89") + return C89; if (std == "c99" || std == "c9x" || std == "iso9899:1999" || std == "iso9899:199x" || std == "gnu99"|| std == "gnu9x") - return "199901L"; + return C99; if (std == "c11" || std == "c1x" || std == "iso9899:2011" || std == "gnu11" || std == "gnu1x") - return "201112L"; + return C11; if (std == "c17" || std == "c18" || std == "iso9899:2017" || std == "iso9899:2018" || std == "gnu17"|| std == "gnu18") - return "201710L"; - if (std == "c23" || std == "gnu23" || std == "c2x" || std == "gnu2x") { - // supported by GCC 9+ and Clang 9+ - // Clang 9, 10, 11, 12, 13 return "201710L" - // Clang 14, 15, 16, 17 return "202000L" - // Clang 9, 10, 11, 12, 13, 14, 15, 16, 17 do not support "c23" and "gnu23" - return "202311L"; + return C17; + if (std == "c23" || std == "gnu23" || std == "c2x" || std == "gnu2x") + return C23; + return CUnknown; +} + +std::string simplecpp::getCStdString(cstd_t std) +{ + switch (std) + { + case C89: + // __STDC_VERSION__ is not set for C90 although the macro was added in the 1994 amendments + return ""; + case C99: + return "199901L"; + case C11: + return "201112L"; + case C17: + return "201710L"; + case C23: + // supported by GCC 9+ and Clang 9+ + // Clang 9, 10, 11, 12, 13 return "201710L" + // Clang 14, 15, 16, 17 return "202000L" + // Clang 9, 10, 11, 12, 13, 14, 15, 16, 17 do not support "c23" and "gnu23" + return "202311L"; + case CUnknown: + return ""; } return ""; } -std::string simplecpp::getCppStdString(const std::string &std) +std::string simplecpp::getCStdString(const std::string &std) +{ + return getCStdString(getCStd(std)); +} + +simplecpp::cppstd_t simplecpp::getCppStd(const std::string &std) { if (std == "c++98" || std == "c++03" || std == "gnu++98" || std == "gnu++03") - return "199711L"; + return CPP03; if (std == "c++11" || std == "gnu++11" || std == "c++0x" || std == "gnu++0x") - return "201103L"; + return CPP11; if (std == "c++14" || std == "c++1y" || std == "gnu++14" || std == "gnu++1y") - return "201402L"; + return CPP14; if (std == "c++17" || std == "c++1z" || std == "gnu++17" || std == "gnu++1z") - return "201703L"; - if (std == "c++20" || std == "c++2a" || std == "gnu++20" || std == "gnu++2a") { - // GCC 10 returns "201703L" - correct in 11+ - return "202002L"; - } - if (std == "c++23" || std == "c++2b" || std == "gnu++23" || std == "gnu++2b") { - // supported by GCC 11+ and Clang 12+ - // GCC 11, 12, 13 return "202100L" - // Clang 12, 13, 14, 15, 16 do not support "c++23" and "gnu++23" and return "202101L" - // Clang 17, 18 return "202302L" - return "202302L"; - } - if (std == "c++26" || std == "c++2c" || std == "gnu++26" || std == "gnu++2c") { - // supported by Clang 17+ - return "202400L"; + return CPP17; + if (std == "c++20" || std == "c++2a" || std == "gnu++20" || std == "gnu++2a") + return CPP20; + if (std == "c++23" || std == "c++2b" || std == "gnu++23" || std == "gnu++2b") + return CPP23; + if (std == "c++26" || std == "c++2c" || std == "gnu++26" || std == "gnu++2c") + return CPP26; + return CPPUnknown; +} + +std::string simplecpp::getCppStdString(cppstd_t std) +{ + switch (std) + { + case CPP03: + return "199711L"; + case CPP11: + return "201103L"; + case CPP14: + return "201402L"; + case CPP17: + return "201703L"; + case CPP20: + // GCC 10 returns "201703L" - correct in 11+ + return "202002L"; + case CPP23: + // supported by GCC 11+ and Clang 12+ + // GCC 11, 12, 13 return "202100L" + // Clang 12, 13, 14, 15, 16 do not support "c++23" and "gnu++23" and return "202101L" + // Clang 17, 18 return "202302L" + return "202302L"; + case CPP26: + // supported by Clang 17+ + return "202400L"; + case CPPUnknown: + return ""; } return ""; } +std::string simplecpp::getCppStdString(const std::string &std) +{ + return getCppStdString(getCppStd(std)); +} + #if (__cplusplus < 201103L) && !defined(__APPLE__) #undef nullptr #endif diff --git a/externals/simplecpp/simplecpp.h b/externals/simplecpp/simplecpp.h index 88a14014567..d6641136dd9 100755 --- a/externals/simplecpp/simplecpp.h +++ b/externals/simplecpp/simplecpp.h @@ -39,6 +39,11 @@ #endif namespace simplecpp { + /** C code standard */ + enum cstd_t { CUnknown=-1, C89, C99, C11, C17, C23 }; + + /** C++ code standard */ + enum cppstd_t { CPPUnknown=-1, CPP03, CPP11, CPP14, CPP17, CPP20, CPP23, CPP26 }; typedef std::string TokenString; class Macro; @@ -368,11 +373,19 @@ namespace simplecpp { /** Convert Cygwin path to Windows path */ SIMPLECPP_LIB std::string convertCygwinToWindowsPath(const std::string &cygwinPath); + /** Returns the C version a given standard */ + SIMPLECPP_LIB cstd_t getCStd(const std::string &std); + + /** Returns the C++ version a given standard */ + SIMPLECPP_LIB cppstd_t getCppStd(const std::string &std); + /** Returns the __STDC_VERSION__ value for a given standard */ SIMPLECPP_LIB std::string getCStdString(const std::string &std); + SIMPLECPP_LIB std::string getCStdString(cstd_t std); /** Returns the __cplusplus value for a given standard */ SIMPLECPP_LIB std::string getCppStdString(const std::string &std); + SIMPLECPP_LIB std::string getCppStdString(cppstd_t std); } #if defined(_MSC_VER) From ab383085604481b6a6898b9f7006afcc6c0e1223 Mon Sep 17 00:00:00 2001 From: firewave Date: Mon, 9 Sep 2024 15:49:06 +0200 Subject: [PATCH 5/5] TestPreprocessor: test handling of standards / added TODOs --- lib/tokenlist.cpp | 2 ++ test/helpers.cpp | 3 +++ test/testpreprocessor.cpp | 45 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index 5356e4917d0..d2a3e84ce8f 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -375,6 +375,8 @@ bool TokenList::createTokensInternal(std::istream &code, const std::string& file // NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved) void TokenList::createTokens(simplecpp::TokenList&& tokenList) { + // TODO: what to do if the list has been filled already? clear mTokensFrontBack? + // tokenList.cfront() might be NULL if the file contained nothing to tokenize so we need to check the files instead if (!tokenList.getFiles().empty()) { // this is a copy diff --git a/test/helpers.cpp b/test/helpers.cpp index 895d924f7cb..633becd1e9b 100644 --- a/test/helpers.cpp +++ b/test/helpers.cpp @@ -170,12 +170,15 @@ void PreprocessorHelper::preprocess(const char code[], std::vector void PreprocessorHelper::preprocess(const char code[], std::vector &files, Tokenizer& tokenizer, ErrorLogger& errorlogger, const simplecpp::DUI& dui) { + // TODO: make sure the given Tokenizer has not been used yet + std::istringstream istr(code); const simplecpp::TokenList tokens1(istr, files, files[0]); // Preprocess.. simplecpp::TokenList tokens2(files); std::map filedata; + // TODO: provide and handle outputList simplecpp::preprocess(tokens2, tokens1, files, filedata, dui); // Tokenizer.. diff --git a/test/testpreprocessor.cpp b/test/testpreprocessor.cpp index f69518911ef..257749f882b 100644 --- a/test/testpreprocessor.cpp +++ b/test/testpreprocessor.cpp @@ -266,6 +266,8 @@ class TestPreprocessor : public TestFixture { TEST_CASE(limitsDefines); TEST_CASE(hashCalculation); + + TEST_CASE(standard); } std::string getConfigsStr(const char filedata[], const char *arg = nullptr) { @@ -2536,6 +2538,49 @@ class TestPreprocessor : public TestFixture { ASSERT(getHash(code) != getHash(code3)); ASSERT(getHash(code2) != getHash(code3)); } + + void standard() { + std::vector files = {"test.cpp"}; + + const char code[] = "int a;"; + // TODO: this bypasses the standard determined from the settings - the parameter should not be exposed + simplecpp::DUI dui; + + { + Tokenizer tokenizer(settingsDefault, *this); + dui.std = "c89"; + PreprocessorHelper::preprocess(code, files, tokenizer, *this, dui); + ASSERT(tokenizer.list.front()); + } + + { + Tokenizer tokenizer(settingsDefault, *this); + dui.std = "gnu23"; + PreprocessorHelper::preprocess(code, files, tokenizer, *this, dui); + ASSERT(tokenizer.list.front()); + } + + { + Tokenizer tokenizer(settingsDefault, *this); + dui.std = "c++98"; + PreprocessorHelper::preprocess(code, files, tokenizer, *this, dui); + ASSERT(tokenizer.list.front()); + } + + { + Tokenizer tokenizer(settingsDefault, *this); + dui.std = "gnu++26"; + PreprocessorHelper::preprocess(code, files, tokenizer, *this, dui); + ASSERT(tokenizer.list.front()); + } + + { + Tokenizer tokenizer(settingsDefault, *this); + dui.std = "gnu77"; + PreprocessorHelper::preprocess(code, files, tokenizer, *this, dui); + ASSERT(!tokenizer.list.front()); // nothing is tokenized when an unknown standard is provided + } + } }; REGISTER_TEST(TestPreprocessor)