Merge pull request #50541 from azat/rust-msan

MSan support for Rust
ClickHouse · Jun 5, 2023 · 5048f24 · 5048f24
2 parents 21e3b6d + bf127f4
commit 5048f24
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 34 deletions.
diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
@@ -46,10 +46,12 @@ ENV CXX=clang++-${LLVM_VERSION}
 # Rust toolchain and libraries
 ENV RUSTUP_HOME=/rust/rustup
 ENV CARGO_HOME=/rust/cargo
-ENV PATH="/rust/cargo/env:${PATH}"
 ENV PATH="/rust/cargo/bin:${PATH}"
 RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
     chmod 777 -R /rust && \
+    rustup toolchain install nightly && \
+    rustup default nightly && \
+    rustup component add rust-src && \
     rustup target add aarch64-unknown-linux-gnu && \
     rustup target add x86_64-apple-darwin && \
     rustup target add x86_64-unknown-freebsd && \

diff --git a/rust/.cargo/config.toml.in b/rust/.cargo/config.toml.in
@@ -1,3 +1,10 @@
 [env]
 CFLAGS = "@RUST_CFLAGS@"
 CXXFLAGS = "@RUST_CXXFLAGS@"
+
+[build]
+rustflags = @RUSTFLAGS@
+rustdocflags = @RUSTFLAGS@
+
+[unstable]
+@RUST_CARGO_BUILD_STD@
diff --git a/rust/BLAKE3/include/blake3.h b/rust/BLAKE3/include/blake3.h
@@ -8,8 +8,6 @@ extern "C" {
 
 char *blake3_apply_shim(const char *begin, uint32_t _size, uint8_t *out_char_data);
 
-char *blake3_apply_shim_msan_compat(const char *begin, uint32_t size, uint8_t *out_char_data);
-
 void blake3_free_char_pointer(char *ptr_to_free);
 
 } // extern "C"

diff --git a/rust/BLAKE3/src/lib.rs b/rust/BLAKE3/src/lib.rs
@@ -3,7 +3,6 @@ extern crate libc;
 
 use std::ffi::{CStr, CString};
 use std::os::raw::c_char;
-use std::mem;
 
 #[no_mangle]
 pub unsafe extern "C" fn blake3_apply_shim(
@@ -24,30 +23,6 @@ pub unsafe extern "C" fn blake3_apply_shim(
     std::ptr::null_mut()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn blake3_apply_shim_msan_compat(
-    mut begin: *const c_char,
-    size: u32,
-    out_char_data: *mut u8,
-) -> *mut c_char {
-    if begin.is_null() {
-        let err_str = CString::new("input was a null pointer").unwrap();
-        return err_str.into_raw();
-    }
-    libc::memset(out_char_data as *mut libc::c_void, 0, mem::size_of::<u8>());
-    let mut hasher = blake3::Hasher::new();
-    let mut vec = Vec::<u8>::new();
-    for _ in 0..size {
-        vec.push(*begin as u8);
-        begin = begin.add(1);
-    }
-    let input_res = vec.as_mut_slice();
-    hasher.update(input_res);
-    let mut reader = hasher.finalize_xof();
-    reader.fill(std::slice::from_raw_parts_mut(out_char_data, blake3::OUT_LEN));
-    std::ptr::null_mut()
-}
-
 // Freeing memory according to docs: https://doc.rust-lang.org/std/ffi/struct.CString.html#method.into_raw
 #[no_mangle]
 pub unsafe extern "C" fn blake3_free_char_pointer(ptr_to_free: *mut c_char) {

diff --git a/rust/CMakeLists.txt b/rust/CMakeLists.txt
@@ -14,8 +14,18 @@ macro(configure_rustc)
         set(RUST_CFLAGS "${RUST_CFLAGS} --sysroot ${CMAKE_SYSROOT}")
     endif()
 
+    set(RUSTFLAGS "[]")
+    set(RUST_CARGO_BUILD_STD "")
+    # For more info: https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html#memorysanitizer
+    if (SANITIZE STREQUAL "memory")
+        set(RUST_CARGO_BUILD_STD "build-std = [\"std\", \"panic_abort\", \"core\", \"alloc\"]")
+        set(RUSTFLAGS "[\"-Zsanitizer=memory\", \"-Zsanitizer-memory-track-origins\"]")
+    endif()
+
     message(STATUS "RUST_CFLAGS: ${RUST_CFLAGS}")
     message(STATUS "RUST_CXXFLAGS: ${RUST_CXXFLAGS}")
+    message(STATUS "RUSTFLAGS: ${RUSTFLAGS}")
+    message(STATUS "RUST_CARGO_BUILD_STD: ${RUST_CARGO_BUILD_STD}")
 
     # NOTE: requires RW access for the source dir
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/.cargo/config.toml.in" "${CMAKE_CURRENT_SOURCE_DIR}/.cargo/config.toml" @ONLY)

diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h
@@ -816,12 +816,7 @@ struct ImplBLAKE3
 #else
     static void apply(const char * begin, const size_t size, unsigned char* out_char_data)
     {
-#    if defined(MEMORY_SANITIZER)
-            auto err_msg = blake3_apply_shim_msan_compat(begin, safe_cast<uint32_t>(size), out_char_data);
-            __msan_unpoison(out_char_data, length);
-#    else
-            auto err_msg = blake3_apply_shim(begin, safe_cast<uint32_t>(size), out_char_data);
-#    endif
+        auto err_msg = blake3_apply_shim(begin, safe_cast<uint32_t>(size), out_char_data);
         if (err_msg != nullptr)
         {
             auto err_st = std::string(err_msg);