Disabling memory sanitizer with one function. Might help with issue 1965

cmake/developer-options.cmake

@@ -50,6 +50,23 @@ undefined behavior.")
   endif()
 endif()
 
+option(SIMDJSON_SANITIZE_MEMORY "Sanitize memory" OFF)
+
+
+if(SIMDJSON_SANITIZE_MEMORY)
+  message(STATUS "Setting the memory sanitizer.")
+  add_compile_options(
+      -fsanitize=memory -fno-sanitize-recover=all
+  )
+  link_libraries(
+      -fsanitize=memory -fno-sanitize-recover=all
+  )
+  # Ubuntu bug for GCC 5.0+ (safe for all versions)
+  if(CMAKE_COMPILER_IS_GNUCC)
+    link_libraries(-fuse-ld=gold)
+  endif()
+endif()
+
 if(SIMDJSON_SANITIZE_THREADS)
   message(STATUS "Setting both the thread sanitizer \
 and the undefined-behavior sanitizer.")

include/simdjson/arm64/bitmanipulation.h

@@ -9,6 +9,10 @@ namespace {
 // but the algorithms do not end up using the returned value.
 // Sadly, sanitizers are not smart enough to figure it out.
 SIMDJSON_NO_SANITIZE_UNDEFINED
+// This function can be used safely even if not all bytes have been
+// initialized.
+// See issue https://github.com/simdjson/simdjson/issues/1965
+SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
 #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long ret;

include/simdjson/haswell/bitmanipulation.h

@@ -9,6 +9,10 @@ namespace {
 // but the algorithms do not end up using the returned value.
 // Sadly, sanitizers are not smart enough to figure it out.
 SIMDJSON_NO_SANITIZE_UNDEFINED
+// This function can be used safely even if not all bytes have been
+// initialized.
+// See issue https://github.com/simdjson/simdjson/issues/1965
+SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
 #if SIMDJSON_REGULAR_VISUAL_STUDIO
   return (int)_tzcnt_u64(input_num);

include/simdjson/icelake/bitmanipulation.h

@@ -9,6 +9,10 @@ namespace {
 // but the algorithms do not end up using the returned value.
 // Sadly, sanitizers are not smart enough to figure it out.
 SIMDJSON_NO_SANITIZE_UNDEFINED
+// This function can be used safely even if not all bytes have been
+// initialized.
+// See issue https://github.com/simdjson/simdjson/issues/1965
+SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
 #if SIMDJSON_REGULAR_VISUAL_STUDIO
   return (int)_tzcnt_u64(input_num);

include/simdjson/portability.h

@@ -148,6 +148,19 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
 #define SIMDJSON_NO_SANITIZE_UNDEFINED
 #endif
 
+
+#if defined(__clang__) || defined(__GNUC__)
+#if defined(__has_feature)
+#  if __has_feature(memory_sanitizer)
+#define SIMDJSON_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory")))
+#  endif // if __has_feature(memory_sanitizer)
+#endif // defined(__has_feature)
+#endif
+// make sure it is defined as 'nothing' if it is unapplicable.
+#ifndef SIMDJSON_NO_SANITIZE_MEMORY
+#define SIMDJSON_NO_SANITIZE_MEMORY
+#endif
+
 #if SIMDJSON_VISUAL_STUDIO
 // This is one case where we do not distinguish between
 // regular visual studio and clang under visual studio.

include/simdjson/ppc64/bitmanipulation.h

@@ -9,6 +9,10 @@ namespace {
 // but the algorithms do not end up using the returned value.
 // Sadly, sanitizers are not smart enough to figure it out.
 SIMDJSON_NO_SANITIZE_UNDEFINED
+// This function can be used safely even if not all bytes have been
+// initialized.
+// See issue https://github.com/simdjson/simdjson/issues/1965
+SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
 #if SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long ret;

include/simdjson/westmere/bitmanipulation.h

@@ -9,6 +9,10 @@ namespace {
 // but the algorithms do not end up using the returned value.
 // Sadly, sanitizers are not smart enough to figure it out.
 SIMDJSON_NO_SANITIZE_UNDEFINED
+// This function can be used safely even if not all bytes have been
+// initialized.
+// See issue https://github.com/simdjson/simdjson/issues/1965
+SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
 #if SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long ret;

tests/dom/document_stream_tests.cpp

@@ -634,7 +634,7 @@ namespace document_stream_tests {
     ASSERT_SUCCESS( odparser.parse_many(json.data(), json.length(), 50).get(odstream) );
     for (auto doc: odstream) {
       if(counter < 6) {
-        int64_t val;
+        int64_t val{};
         ASSERT_SUCCESS(doc.at_pointer("/4").get(val));
         ASSERT_EQUAL(val, 5);
       } else {

tests/ondemand/ondemand_document_stream_tests.cpp

@@ -7,7 +7,7 @@ namespace document_stream_tests {
 
     template <typename T>
     bool process_doc(T &docref) {
-        int64_t val;
+        int64_t val{};
         ASSERT_SUCCESS(docref.at_pointer("/4").get(val));
         ASSERT_EQUAL(val, 5);
         return true;
@@ -604,7 +604,7 @@ namespace document_stream_tests {
         ASSERT_SUCCESS( odparser.iterate_many(json.data(), json.length(), 50).get(odstream) );
         for (auto doc: odstream) {
             if(counter < 6) {
-                int64_t val;
+                int64_t val{};
                 ASSERT_SUCCESS(doc.at_pointer("/4").get(val));
                 ASSERT_EQUAL(val, 5);
             } else {

tests/ondemand/ondemand_object_tests.cpp

@@ -6,6 +6,7 @@ using namespace simdjson;
 namespace object_tests {
   using namespace std;
   using simdjson::ondemand::json_type;
+
   bool issue1745() {
     TEST_START();
     auto json = R"({
@@ -225,6 +226,25 @@ namespace object_tests {
   }
 
 #if SIMDJSON_EXCEPTIONS
+
+  bool issue1965() {
+    TEST_START();
+    std::string str = "{\"query\":\"ah\"}";
+    std::unique_ptr<char[]> buffer(new char[str.size() + simdjson::SIMDJSON_PADDING]);
+    memcpy(buffer.get(), str.data(), str.size());
+    simdjson::padded_string_view view(buffer.get(), str.size(), str.size() + simdjson::SIMDJSON_PADDING);
+    simdjson::ondemand::parser parser;
+    simdjson::ondemand::document doc = parser.iterate(view);
+    simdjson::ondemand::object root = doc.get_object();
+    simdjson::ondemand::value query = root.find_field("query");
+    simdjson::ondemand::raw_json_string raw = query.get_raw_json_string();
+    std::unique_ptr<uint8_t[]> dst_buffer(new uint8_t[3 + simdjson::SIMDJSON_PADDING]);
+    uint8_t * dst = dst_buffer.get();
+    std::string_view fieldstring = parser.unescape(raw, dst);
+    std::cout << fieldstring << std::endl;
+    TEST_SUCCEED();
+  }
+
   bool issue1745_with_exceptions() {
     TEST_START();
     auto json = R"({
@@ -1221,6 +1241,9 @@ namespace object_tests {
 
   bool run() {
     return
+#if SIMDJSON_EXCEPTIONS
+           issue1965() &&
+#endif
            issue1876a() &&
            issue1876() &&
            test_strager() &&