Make the Parser independent from the global C-locale (#5028)

* Make the Parser independent from the global C-locale

* Set a specific test locale using the environment variable FLATBUFFERS_TEST_LOCALE

* Remove redundant static qualifiers

.travis.yml

@@ -2,6 +2,12 @@ env:
   global:
     # Set at the root level as this is ignored when set under matrix.env.
     - GCC_VERSION="4.9"
+    # Fail on first error if UBSAN or ASAN enabled for a target
+    - UBSAN_OPTIONS=halt_on_error=1
+    - ASAN_OPTIONS=halt_on_error=1
+    # Travis machines have 2 cores
+    - JOBS=2
+    - MAKEFLAGS="-j 2"
 
 conan-linux: &conan-linux
   os: linux
@@ -53,7 +59,7 @@ matrix:
     #        branch: master
     - language: cpp
       os:
-      - linux
+        - linux
 
       compiler:
         - gcc
@@ -79,8 +85,8 @@ matrix:
         -DGRPC_INSTALL_PATH=$TRAVIS_BUILD_DIR/google/grpc/install 
         -DPROTOBUF_DOWNLOAD_PATH=$TRAVIS_BUILD_DIR/google/grpc/third_party/protobuf
         -DFLATBUFFERS_CODE_SANITIZE=ON
-      - make
-      - LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/google/grpc/install/lib make test ARGS=-V
+      - cmake --build . -- -j${JOBS}
+      - LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/google/grpc/install/lib ctest --extra-verbose --output-on-failure
       - bash .travis/check-generate-code.sh
       - if [ "$CONAN" == "true" ] && [ "$TRAVIS_OS_NAME" == "linux" ]; then sudo pip install conan && conan create . flatbuffers/${TRAVIS_BRANCH}@google/testing -s build_type=$BUILD_TYPE -tf conan/test_package; fi
 
@@ -91,6 +97,7 @@ matrix:
         matrix:
           - BUILD_TYPE=Debug
           - BUILD_TYPE=Release
+
       script:
       - bash grpc/build_grpc.sh
       - cmake .
@@ -99,10 +106,9 @@ matrix:
         -DGRPC_INSTALL_PATH=$TRAVIS_BUILD_DIR/google/grpc/install 
         -DPROTOBUF_DOWNLOAD_PATH=$TRAVIS_BUILD_DIR/google/grpc/third_party/protobuf
         -DFLATBUFFERS_CODE_SANITIZE=ON
-      - make
-      - ./flattests
+      - cmake --build . -- -j${JOBS}
+      - DYLD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/google/grpc/install/lib ctest --extra-verbose --output-on-failure
       - bash .travis/check-generate-code.sh
-      - DYLD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/google/grpc/install/lib ./grpctest
 
     - <<: *conan-linux
       env: CONAN_GCC_VERSIONS=4.9 CONAN_DOCKER_IMAGE=conanio/gcc49
@@ -149,6 +155,7 @@ matrix:
           - extra-android-m2repository
       compiler:
         - gcc
+
       before_install:
         - git clone https://github.com/urho3d/android-ndk.git $HOME/android-ndk-root
         - export ANDROID_NDK_HOME=$HOME/android-ndk-root

CMakeLists.txt

@@ -1,6 +1,7 @@
 cmake_minimum_required(VERSION 2.8)
 # generate compile_commands.json
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+include(CheckCXXSymbolExists)
 
 project(FlatBuffers)
 
@@ -35,6 +36,16 @@ if(DEFINED FLATBUFFERS_MAX_PARSING_DEPTH)
   message(STATUS "FLATBUFFERS_MAX_PARSING_DEPTH: ${FLATBUFFERS_MAX_PARSING_DEPTH}")
 endif()
 
+# Auto-detect locale-narrow 'strtod_l' function.
+if(NOT DEFINED FLATBUFFERS_LOCALE_INDEPENDENT)
+  if(MSVC)
+    check_cxx_symbol_exists(_strtof_l stdlib.h FLATBUFFERS_LOCALE_INDEPENDENT)
+  else()
+    check_cxx_symbol_exists(strtof_l stdlib.h FLATBUFFERS_LOCALE_INDEPENDENT)
+  endif()
+endif()
+add_definitions(-DFLATBUFFERS_LOCALE_INDEPENDENT=$<BOOL:${FLATBUFFERS_LOCALE_INDEPENDENT}>)
+
 set(FlatBuffers_Library_SRCS
   include/flatbuffers/code_generators.h
   include/flatbuffers/base.h
@@ -213,6 +224,7 @@ function(add_fsanitize_to_target _target _sanitizer)
     target_link_libraries(${_target} PRIVATE
       "-fsanitize${_sanitizer_flags}")
     set_property(TARGET ${_target} PROPERTY POSITION_INDEPENDENT_CODE ON)
+    message(STATUS "Sanitizer ${_sanitizer_flags} added to ${_target}")
   endif()
 endfunction()
 

docs/source/CppUsage.md

@@ -499,11 +499,47 @@ To use scalars, simply wrap them in a struct.
 
 ## Depth limit of nested objects and stack-overflow control
 The parser of Flatbuffers schema or json-files is kind of recursive parser.
-To avoid stack-overflow problem the parser has a built-in limiter of recursion depth.
-Number of nested declarations in a schema or number of nested json-objects is limited. 
-By default, this depth limit set to `64`.
-It is possible to override this limit with `FLATBUFFERS_MAX_PARSING_DEPTH` definition.
-This definition can be helpful for testing purposes or embedded applications.
-For details see [build](@ref flatbuffers_guide_building) of CMake-based projects.
+To avoid stack-overflow problem the parser has a built-in limiter of 
+recursion depth. Number of nested declarations in a schema or number of 
+nested json-objects is limited. By default, this depth limit set to `64`.
+It is possible to override this limit with `FLATBUFFERS_MAX_PARSING_DEPTH` 
+definition. This definition can be helpful for testing purposes or embedded 
+applications. For details see [build](@ref flatbuffers_guide_building) of 
+CMake-based projects.
+
+## Dependence from C-locale {#flatbuffers_locale_cpp}
+The Flatbuffers [grammar](@ref flatbuffers grammar) uses ASCII 
+character set for identifiers, alphanumeric literals, reserved words.
+
+Internal implementation of the Flatbuffers depends from functions which 
+depend from C-locale: `strtod()` or `strtof()`, for example.
+The library expects the dot `.` symbol as the separator of an integer 
+part from the fractional part of a float number.
+Another separator symbols (`,` for example) will break the compatibility 
+and may lead to an error while parsing a Flatbuffers schema or a json file.
+
+The Standard C locale is a global resource, there is only one locale for 
+the entire application. Some modern compilers and platforms have 
+locale-independent or locale-narrow functions `strtof_l`, `strtod_l`, 
+`strtoll_l`, `strtoull_l` to resolve this dependency. 
+These functions use specified locale rather than the global or per-thread 
+locale instead. They are part of POSIX-2008 but not part of the C/C++ 
+standard library, therefore, may be missing on some platforms.
+
+The Flatbuffers library try to detect these functions at configuration and 
+compile time:
+- `_MSC_VER >= 1900`: check MSVC2012 or higher for MSVC buid
+- `_XOPEN_SOURCE>=700`: check POSIX-2008 for GCC/Clang build
+- `check_cxx_symbol_exists(strtof_l stdlib.h)`: CMake check of `strtod_f`
+
+After detection, the definition `FLATBUFFERS_LOCALE_INDEPENDENT` will be 
+set to `0` or `1`.
+
+It is possible to test the compatibility of the Flatbuffers library with 
+a specific locale using the environment variable `FLATBUFFERS_TEST_LOCALE`:
+```sh
+>FLATBUFFERS_TEST_LOCALE="" ./flattests
+>FLATBUFFERS_TEST_LOCALE="ru_RU.CP1251" ./flattests
+```
 
 <br>

include/flatbuffers/base.h

@@ -195,15 +195,35 @@
   #endif
 #endif // !FLATBUFFERS_HAS_NEW_STRTOD
 
-// Suppress sanitizer directives.
+#ifndef FLATBUFFERS_LOCALE_INDEPENDENT
+  // Enable locale independent functions {strtof_l, strtod_l,strtoll_l, strtoull_l}.
+  // They are part of the POSIX-2008 but not part of the C/C++ standard.
+  // GCC/Clang have definition (_XOPEN_SOURCE>=700) if POSIX-2008.
+  #if ((defined(_MSC_VER) && _MSC_VER >= 1800)            || \
+       (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE>=700)))
+    #define FLATBUFFERS_LOCALE_INDEPENDENT 1
+  #else
+    #define FLATBUFFERS_LOCALE_INDEPENDENT 0
+  #endif
+#endif  // !FLATBUFFERS_LOCALE_INDEPENDENT
+
+// Suppress Undefined Behavior Sanitizer (recoverable only). Usage:
+// - __supress_ubsan__("undefined")
+// - __supress_ubsan__("signed-integer-overflow")
 #if defined(__clang__)
-  #define __no_sanitize_undefined__(reason) __attribute__((no_sanitize("undefined")))
+  #define __supress_ubsan__(type) __attribute__((no_sanitize(type)))
 #elif defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 408)
-  #define __no_sanitize_undefined__(reason) __attribute__((no_sanitize_undefined))
+  #define __supress_ubsan__(type) __attribute__((no_sanitize_undefined))
 #else
-  #define __no_sanitize_undefined__(reason)
+  #define __supress_ubsan__(type)
 #endif
 
+// This is constexpr function used for checking compile-time constants.
+// Avoid `#pragma warning(disable: 4127) // C4127: expression is constant`.
+template<typename T> FLATBUFFERS_CONSTEXPR inline bool IsConstTrue(T t) {
+  return !!t;
+}
+
 /// @endcond
 
 /// @file
@@ -287,13 +307,15 @@ template<typename T> T EndianScalar(T t) {
 }
 
 template<typename T>
-__no_sanitize_undefined__("C++ aliasing type rules, see std::bit_cast<>")
+// UBSAN: C++ aliasing type rules, see std::bit_cast<> for details.
+__supress_ubsan__("alignment")
 T ReadScalar(const void *p) {
   return EndianScalar(*reinterpret_cast<const T *>(p));
 }
 
 template<typename T>
-__no_sanitize_undefined__("C++ aliasing type rules, see std::bit_cast<>")
+// UBSAN: C++ aliasing type rules, see std::bit_cast<> for details.
+__supress_ubsan__("alignment")
 void WriteScalar(void *p, T t) {
   *reinterpret_cast<T *>(p) = EndianScalar(t);
 }