From 21961806dc7be4588c68f8016b6593ea27f710a2 Mon Sep 17 00:00:00 2001 From: Marton Greber Date: Wed, 18 Nov 2020 14:10:36 -0600 Subject: [PATCH] KUDU-3374 Add support for M1 and macOS Monterey The macOS Monterey OS upgrade broke glog 0.3.5, moreover building Kudu with LLVM 9 on Apple silicon is not really feasible. There has been multiple non-merged patches submitted to tackle these issues: Upgrade glog to 0.6.0 [1] Fix codegen build on MacOS Monterey [2] Fix building on Apple M1 [3] Upgrade to LLVM 11 and IWYU 0.15 [4] This patch squashes all of the above and provides the necessary glue changes. LLVM is updated to version 11 and glog to version 0.6.0 to fix the initial build issues. Building the glog tests has to be turned off as it causes linker error in TSAN build. The optional ZLIB dependency in LLVM-IWYU is removed as the original issue -mentioned in the comments- has been resolved, and this caused build issues on Unix. [5] The homebrew prefix is changed from /usr/local to /opt/homebrew on ARM macs. In places where package locations are provided, an ARM alternative location has been added. Added a patch [6] to fix null pointer dereference in rapidjson. Added another patch [7] containing assertions to a similar suppress clang warnings in rapidjson. Building tests in glog has to be turned off [8] as it causes linker error in tsan build. With the clang upgrade it now links against libatomic in TSAN builds. In dist-test.py libatomic is added to the list of shipped libraries as it was missing on the target machines. A couple of new TSAN races came up with LLVM regexes, added those to the sanitizer suppression list. [1] Change-Id: https://gerrit.cloudera.org/#/c/18460/ [2] Change-Id: https://gerrit.cloudera.org/#/c/18461/ [3] Change-Id: https://gerrit.cloudera.org/#/c/18464/ [4] Change-Id: https://gerrit.cloudera.org/#/c/16768/ [5] https://github.com/include-what-you-use/include-what-you-use/ issues/539 [6] https://github.com/Tencent/rapidjson/pull/727 [7] https://github.com/Tencent/rapidjson/pull/757 [8] https://github.com/google/glog/issues/54 Change-Id: I9877f95340b969308c317a6bac50665ff78e329e Reviewed-on: http://gerrit.cloudera.org:8080/18770 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin --- CMakeLists.txt | 1 + build-support/dist_test.py | 4 + build-support/iwyu/mappings/glog.imp | 7 +- cmake_modules/FindKerberosPrograms.cmake | 2 + src/kudu/codegen/CMakeLists.txt | 2 +- src/kudu/codegen/code_generator.cc | 6 +- src/kudu/codegen/module_builder.cc | 7 +- src/kudu/common/row_operations-test.cc | 5 +- src/kudu/fs/dir_util-test.cc | 8 +- src/kudu/gutil/atomicops-internals-macosx.h | 10 +- src/kudu/gutil/dynamic_annotations.c | 2 +- src/kudu/gutil/dynamic_annotations.h | 2 +- src/kudu/ranger/ranger_client.cc | 2 +- src/kudu/security/test/mini_kdc.cc | 2 + src/kudu/server/diagnostics_log.cc | 3 +- src/kudu/server/pprof_path_handlers.cc | 3 +- src/kudu/tablet/concurrent_btree.h | 20 +- src/kudu/util/async_logger.cc | 2 +- src/kudu/util/async_logger.h | 2 +- src/kudu/util/debug-util.cc | 2 +- src/kudu/util/debug/trace_logging.h | 6 +- src/kudu/util/flags.cc | 2 +- src/kudu/util/logging-test.cc | 7 +- src/kudu/util/logging.cc | 13 +- src/kudu/util/logging.h | 6 +- src/kudu/util/logging_test_util.h | 4 +- src/kudu/util/minidump.cc | 4 +- src/kudu/util/rw_mutex-test.cc | 56 ++--- src/kudu/util/sanitizer_options.cc | 7 + thirdparty/build-definitions.sh | 71 +++--- thirdparty/build-thirdparty.sh | 12 +- thirdparty/download-thirdparty.sh | 28 +-- thirdparty/package-llvm.sh | 11 +- .../patches/glog-fix-symbolization.patch | 212 ------------------ .../glog-issue-198-fix-unused-warnings.patch | 47 ---- .../glog-issue-54-dont-build-tests.patch | 147 ------------ .../patches/glog-make-internals-visible.patch | 23 ++ .../glog-support-stacktrace-for-aarch64.patch | 33 +-- ...mock-remove-unused-gunit-iwyu-pragma.patch | 11 + .../patches/llvm-iwyu-include-picker.patch | 10 - .../llvm-iwyu-sized-deallocation.patch | 116 ---------- ...idjson-assertions-for-clang-warnings.patch | 21 ++ ...d-pointer-arithmetic-on-null-pointer.patch | 43 ++++ thirdparty/preflight.py | 7 +- thirdparty/vars.sh | 6 +- 45 files changed, 291 insertions(+), 704 deletions(-) delete mode 100644 thirdparty/patches/glog-fix-symbolization.patch delete mode 100644 thirdparty/patches/glog-issue-198-fix-unused-warnings.patch delete mode 100644 thirdparty/patches/glog-issue-54-dont-build-tests.patch create mode 100644 thirdparty/patches/glog-make-internals-visible.patch create mode 100644 thirdparty/patches/gmock-remove-unused-gunit-iwyu-pragma.patch delete mode 100644 thirdparty/patches/llvm-iwyu-include-picker.patch delete mode 100644 thirdparty/patches/llvm-iwyu-sized-deallocation.patch create mode 100644 thirdparty/patches/rapidjson-assertions-for-clang-warnings.patch create mode 100644 thirdparty/patches/rapidjson-avoid-pointer-arithmetic-on-null-pointer.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ef5877db4..49d7082e10 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -165,6 +165,7 @@ if ("$ENV{NO_REBUILD_THIRDPARTY}" STREQUAL "") endif() endif() + ############################################################ # Compiler flags ############################################################ diff --git a/build-support/dist_test.py b/build-support/dist_test.py index 00d4c786aa..d30f15f2e1 100755 --- a/build-support/dist_test.py +++ b/build-support/dist_test.py @@ -240,6 +240,10 @@ def is_lib_whitelisted(lib): # installed versions different from the dist_test image. if "libcrypto" in lib or "libsasl2" in lib or "libssl" in lib: return True + # After upgrading to LLVM 11, TSAN builds link libatomic. + # Since it was not present on the test machines, it is whitelisted. + if "libatomic" in lib: + return True return False return True diff --git a/build-support/iwyu/mappings/glog.imp b/build-support/iwyu/mappings/glog.imp index 08c5e3529b..0cf6a06ede 100644 --- a/build-support/iwyu/mappings/glog.imp +++ b/build-support/iwyu/mappings/glog.imp @@ -23,5 +23,10 @@ { symbol: [ "CHECK_GE", private, "", public ] }, { symbol: [ "CHECK_GT", private, "", public ] }, { symbol: [ "ErrnoLogMessage", private, "", public ] }, - { symbol: [ "COMPACT_GOOGLE_LOG_0", private, "", public ] } + { symbol: [ "COMPACT_GOOGLE_LOG_0", private, "", public ] }, + +# Upgrading from glog 0.3.5 to 0.6.0 the includes for log_severity.h and vlog_is_on.h +# are changed from #include "" to #include <> which seem to cause issues. + { include: ["", "public", "", "public"] }, + { include: ["", "public", "", "public"] } ] diff --git a/cmake_modules/FindKerberosPrograms.cmake b/cmake_modules/FindKerberosPrograms.cmake index f5c0b3f261..76e20848e9 100644 --- a/cmake_modules/FindKerberosPrograms.cmake +++ b/cmake_modules/FindKerberosPrograms.cmake @@ -28,6 +28,8 @@ foreach(bin ${bins}) /usr/sbin # Homebrew install location. /usr/local/opt/krb5/sbin + # Homebrew arm install location + /opt/homebrew/opt/krb5/sbin # Macports install location. /opt/local/sbin # SLES diff --git a/src/kudu/codegen/CMakeLists.txt b/src/kudu/codegen/CMakeLists.txt index 19ef2f5de2..c87b0047a5 100644 --- a/src/kudu/codegen/CMakeLists.txt +++ b/src/kudu/codegen/CMakeLists.txt @@ -69,7 +69,7 @@ if (APPLE) # Clang does not know about by default. set(PREFIXED_IR_INCLUDES ${PREFIXED_IR_INCLUDES} - -cxx-isystem "/Library/Developer/CommandLineTools/usr/include/c++/v1") + -cxx-isystem "${CMAKE_OSX_SYSROOT}") endif() # Get preprocessing definitions, which enable directives for glog and gtest. diff --git a/src/kudu/codegen/code_generator.cc b/src/kudu/codegen/code_generator.cc index 7e419abdc8..642b84172f 100644 --- a/src/kudu/codegen/code_generator.cc +++ b/src/kudu/codegen/code_generator.cc @@ -158,14 +158,14 @@ int DumpAsm(FuncPtr fptr, const TargetMachine& tm, std::ostream* out, int max_in MCInst inst; uint64_t size; MCDisassembler::DecodeStatus stat = - disas->getInstruction(inst, size, mem_obj.slice(addr), addr, llvm::nulls(), llvm::nulls()); + disas->getInstruction(inst, size, mem_obj.slice(addr), addr, llvm::nulls()); if (stat != MCDisassembler::Success) { *out << "\n" << std::dec; } else { string annotations; - printer->printInst(&inst, os, annotations, subtarget_info); + printer->printInst(&inst, addr, annotations, subtarget_info, os); os << " " << annotations << "\n"; // We need to check the opcode name for "RET" instead of comparing // the opcode to llvm::ReturnInst::getOpcode() because the native @@ -178,7 +178,7 @@ int DumpAsm(FuncPtr fptr, const TargetMachine& tm, std::ostream* out, int max_in // LLVM RTTI, since subclassing an LLVM interface would require // identical RTTI settings between LLVM and Kudu (see: // http://llvm.org/docs/Packaging.html#c-features). - string opname = printer->getOpcodeName(inst.getOpcode()); + string opname = printer->getOpcodeName(inst.getOpcode()).str(); std::transform(opname.begin(), opname.end(), opname.begin(), ::toupper); if (opname.find("RET") != string::npos) return i + 1; } diff --git a/src/kudu/codegen/module_builder.cc b/src/kudu/codegen/module_builder.cc index e0818de6b1..59f3ec4afd 100644 --- a/src/kudu/codegen/module_builder.cc +++ b/src/kudu/codegen/module_builder.cc @@ -29,7 +29,10 @@ // for successful run-time operation of the code generator. #include #include +#include #include +#include +#include #include #include // IWYU pragma: keep #include @@ -125,7 +128,7 @@ string ToString(const Module& m) { // This method is needed for the implicit conversion from // llvm::StringRef to std::string string ToString(const Function* f) { - return f->getName(); + return f->getName().str(); } bool ModuleContains(const Module& m, const Function* fptr) { @@ -379,7 +382,7 @@ TargetMachine* ModuleBuilder::GetTargetMachine() const { unordered_set ModuleBuilder::GetFunctionNames() const { unordered_set ret; for (const JITFuture& fut : futures_) { - ret.insert(CHECK_NOTNULL(fut.llvm_f_)->getName()); + ret.insert(CHECK_NOTNULL(fut.llvm_f_)->getName().str()); } return ret; } diff --git a/src/kudu/common/row_operations-test.cc b/src/kudu/common/row_operations-test.cc index d501180870..271220ea56 100644 --- a/src/kudu/common/row_operations-test.cc +++ b/src/kudu/common/row_operations-test.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -298,7 +299,7 @@ TEST_F(RowOperationsTest, SchemaFuzz) { g_failing_case.server_schema = &server_schema; g_failing_case.row = &row; ASAN_SET_DEATH_CALLBACK(&DumpFailingCase); - google::InstallFailureFunction(&GlogFailure); + google::InstallFailureFunction(reinterpret_cast(GlogFailure)); for (int i = 0; i < client_schema.num_columns(); i++) { if (client_schema.column(i).is_nullable() && @@ -326,7 +327,7 @@ TEST_F(RowOperationsTest, SchemaFuzz) { DoFuzzTest(server_schema, row, 100); ASAN_SET_DEATH_CALLBACK(NULL); - google::InstallFailureFunction(&abort); + google::InstallFailureFunction(reinterpret_cast(abort)); } } diff --git a/src/kudu/fs/dir_util-test.cc b/src/kudu/fs/dir_util-test.cc index d669188eee..07bcb094ef 100644 --- a/src/kudu/fs/dir_util-test.cc +++ b/src/kudu/fs/dir_util-test.cc @@ -82,15 +82,15 @@ TEST_F(KuduTest, Locking) { // Note: we must use a death test here because file locking is only // disallowed across processes, and death tests spawn child processes. - ASSERT_DEATH({ + ASSERT_DEATH(({ DirInstanceMetadataFile second(env_, "", kType, kFileName); CHECK_OK(second.LoadFromDisk()); CHECK_EQ(kUuid, second.uuid()); CHECK_OK(second.Lock()); - }, "Could not lock"); + }), "Could not lock"); ASSERT_OK(first.Unlock()); - ASSERT_DEATH({ + ASSERT_DEATH(({ DirInstanceMetadataFile second(env_, "", kType, kFileName); CHECK_OK(second.LoadFromDisk()); CHECK_EQ(kUuid, second.uuid()); @@ -100,7 +100,7 @@ TEST_F(KuduTest, Locking) { } else { LOG(FATAL) << "Could not lock: " << s.ToString(); } - }, "Lock successfully acquired"); + }), "Lock successfully acquired"); } } // namespace fs diff --git a/src/kudu/gutil/atomicops-internals-macosx.h b/src/kudu/gutil/atomicops-internals-macosx.h index a260d3b6a1..4b12451e9a 100644 --- a/src/kudu/gutil/atomicops-internals-macosx.h +++ b/src/kudu/gutil/atomicops-internals-macosx.h @@ -270,11 +270,15 @@ inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { *ptr = value; } -// Issue the x86 "pause" instruction, which tells the CPU that we -// are in a spinlock wait loop and should allow other hyperthreads -// to run, not speculate memory access, etc. +// Issue the x86 "pause" instruction (or "yield" on aarch64), which +// tells the CPU that we are in a spinlock wait loop and should allow +// other hyperthreads to run, not speculate memory access, etc. inline void PauseCPU() { +#ifdef __aarch64__ + __asm__ __volatile__("yield" : : : "memory"); +#else __asm__ __volatile__("pause" : : : "memory"); +#endif } inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) { diff --git a/src/kudu/gutil/dynamic_annotations.c b/src/kudu/gutil/dynamic_annotations.c index f93b634bd1..2bd8662993 100644 --- a/src/kudu/gutil/dynamic_annotations.c +++ b/src/kudu/gutil/dynamic_annotations.c @@ -113,7 +113,7 @@ void AnnotateBenignRace(const char *file, int line, const char *description){} void AnnotateBenignRaceSized(const char *file, int line, const volatile void *mem, - long size, + size_t size, const char *description) {} void AnnotateMutexIsUsedAsCondVar(const char *file, int line, const volatile void *mu){} diff --git a/src/kudu/gutil/dynamic_annotations.h b/src/kudu/gutil/dynamic_annotations.h index 5cc5abaedf..b819464bc8 100644 --- a/src/kudu/gutil/dynamic_annotations.h +++ b/src/kudu/gutil/dynamic_annotations.h @@ -522,7 +522,7 @@ void AnnotateBenignRace(const char *file, int line, const char *description); void AnnotateBenignRaceSized(const char *file, int line, const volatile void *address, - long size, + size_t size, const char *description); void AnnotateMutexIsUsedAsCondVar(const char *file, int line, const volatile void *mu); diff --git a/src/kudu/ranger/ranger_client.cc b/src/kudu/ranger/ranger_client.cc index eead456c1d..54d8cc99e7 100644 --- a/src/kudu/ranger/ranger_client.cc +++ b/src/kudu/ranger/ranger_client.cc @@ -99,7 +99,7 @@ TAG_FLAG(ranger_logtostdout, advanced); TAG_FLAG(ranger_logtostdout, evolving); DECLARE_int32(max_log_files); -DECLARE_int32(max_log_size); +DECLARE_uint32(max_log_size); DECLARE_string(log_dir); METRIC_DEFINE_histogram(server, ranger_subprocess_execution_time_ms, diff --git a/src/kudu/security/test/mini_kdc.cc b/src/kudu/security/test/mini_kdc.cc index 831699c4d2..a80b67b372 100644 --- a/src/kudu/security/test/mini_kdc.cc +++ b/src/kudu/security/test/mini_kdc.cc @@ -108,6 +108,8 @@ Status GetBinaryPath(const string& binary, string* path) { static const vector kCommonLocations = { "/usr/local/opt/krb5/sbin", // Homebrew "/usr/local/opt/krb5/bin", // Homebrew + "/opt/homebrew/opt/krb5/sbin", // Homebrew arm + "/opt/homebrew/opt/krb5/bin", // Homebrew arm "/opt/local/sbin", // Macports "/opt/local/bin", // Macports "/usr/lib/mit/sbin", // SLES diff --git a/src/kudu/server/diagnostics_log.cc b/src/kudu/server/diagnostics_log.cc index 03a8a1b7cf..702d1e3ab0 100644 --- a/src/kudu/server/diagnostics_log.cc +++ b/src/kudu/server/diagnostics_log.cc @@ -17,6 +17,7 @@ #include "kudu/server/diagnostics_log.h" +#include #include #include #include @@ -65,7 +66,7 @@ namespace google { // symbol name to "out". The symbol name is demangled if possible // (supports symbols generated by GCC 3.x or newer). Otherwise, // returns false. -bool Symbolize(void *pc, char *out, int out_size); +bool Symbolize(void *pc, char *out, size_t out_size); } DEFINE_int32(diagnostics_log_stack_traces_interval_ms, 60000, diff --git a/src/kudu/server/pprof_path_handlers.cc b/src/kudu/server/pprof_path_handlers.cc index 971eb292e5..3b809218d8 100644 --- a/src/kudu/server/pprof_path_handlers.cc +++ b/src/kudu/server/pprof_path_handlers.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -66,7 +67,7 @@ namespace google { // symbol name to "out". The symbol name is demangled if possible // (supports symbols generated by GCC 3.x or newer). Otherwise, // returns false. -bool Symbolize(void *pc, char *out, int out_size); +bool Symbolize(void *pc, char *out, size_t out_size); } namespace kudu { diff --git a/src/kudu/tablet/concurrent_btree.h b/src/kudu/tablet/concurrent_btree.h index 2084af4070..2965c332ff 100644 --- a/src/kudu/tablet/concurrent_btree.h +++ b/src/kudu/tablet/concurrent_btree.h @@ -90,6 +90,10 @@ struct BTreeTraits { template inline void PrefetchMemory(const T *addr) { + // During the LLVM 11 upgrade, the sanitizer runs reported null pointer dereference. Wrapping the + // prefetch() call with an if statement fixed the errors. In the end, it was decided to add a + // DCHECK() here, and wrap all uses of PrefetchMemory() with the null pointer guard. + DCHECK(addr); int size = std::min(sizeof(T), 4 * CACHELINE_SIZE); for (int i = 0; i < size; i += CACHELINE_SIZE) { @@ -1168,7 +1172,9 @@ class CBTree { while (node.type() != NodePtr::LEAF_NODE) { #ifdef TRAVERSE_PREFETCH - PrefetchMemory(node.internal_node_ptr()); + if (node.internal_node_ptr()) { + PrefetchMemory(node.internal_node_ptr()); + } #endif retry_in_node: int num_children = node.internal_node_ptr()->num_children_; @@ -1206,7 +1212,9 @@ class CBTree { version = child_version; } #ifdef TRAVERSE_PREFETCH - PrefetchMemory(node.leaf_node_ptr()); + if (node.leaf_node_ptr()) { + PrefetchMemory(node.leaf_node_ptr()); + } #endif *stable_version = version; return node.leaf_node_ptr(); @@ -1777,7 +1785,9 @@ class CBTreeIterator { AtomicVersion version; LeafNode *leaf = tree_->TraverseToLeaf(key, &version); #ifdef SCAN_PREFETCH - PrefetchMemory(leaf->next_); + if (leaf->next_) { + PrefetchMemory(leaf->next_); + } #endif // If the tree is frozen, we don't need to follow optimistic concurrency. @@ -1815,7 +1825,9 @@ class CBTreeIterator { return false; } #ifdef SCAN_PREFETCH - PrefetchMemory(next->next_); + if (next->next_) { + PrefetchMemory(next->next_); + } for (int i = 0; i < next->num_entries(); i++) { next->vals_[i].prefetch(); } diff --git a/src/kudu/util/async_logger.cc b/src/kudu/util/async_logger.cc index 3214a42c8c..be1678938e 100644 --- a/src/kudu/util/async_logger.cc +++ b/src/kudu/util/async_logger.cc @@ -61,7 +61,7 @@ void AsyncLogger::Stop() { void AsyncLogger::Write(bool force_flush, time_t timestamp, const char* message, - int message_len) { + size_t message_len) { { MutexLock l(lock_); DCHECK_EQ(state_, RUNNING); diff --git a/src/kudu/util/async_logger.h b/src/kudu/util/async_logger.h index aedbddef05..48a304657d 100644 --- a/src/kudu/util/async_logger.h +++ b/src/kudu/util/async_logger.h @@ -86,7 +86,7 @@ class AsyncLogger : public google::base::Logger { void Write(bool force_flush, time_t timestamp, const char* message, - int message_len) override; + size_t message_len) override; // Flush any buffered messages. void Flush() override; diff --git a/src/kudu/util/debug-util.cc b/src/kudu/util/debug-util.cc index 44a44aa137..3db8afd97a 100644 --- a/src/kudu/util/debug-util.cc +++ b/src/kudu/util/debug-util.cc @@ -108,7 +108,7 @@ extern int GetStackTrace(void** result, int max_depth, int skip_count); // symbol name to "out". The symbol name is demangled if possible // (supports symbols generated by GCC 3.x or newer). Otherwise, // returns false. -bool Symbolize(void *pc, char *out, int out_size); +bool Symbolize(void *pc, char *out, size_t out_size); namespace glog_internal_namespace_ { extern void DumpStackTraceToString(string *s); diff --git a/src/kudu/util/debug/trace_logging.h b/src/kudu/util/debug/trace_logging.h index 2f0fec944c..90a9c4fd1c 100644 --- a/src/kudu/util/debug/trace_logging.h +++ b/src/kudu/util/debug/trace_logging.h @@ -102,9 +102,9 @@ class TraceGLog { class TraceLogSink : public google::LogSink { public: explicit TraceLogSink(const char* category) : category_(category) {} - void send(google::LogSeverity severity, const char* full_filename, + void send(google::LogSeverity severity, const char* /*full_filename*/, const char* base_filename, int line, - const struct ::tm* tm_time, const char* message, + const ::google::LogMessageTime& logmsgtime, const char* message, size_t message_len) override { // Rather than calling TRACE_EVENT_INSTANT here, we have to do it from // the destructor. This is because glog holds its internal mutex while @@ -116,7 +116,7 @@ class TraceGLog { // we defer the tracing until the google::LogMessage has destructed and the // glog lock is available again. str_ = ToString(severity, base_filename, line, - tm_time, message, message_len); + logmsgtime, message, message_len); } virtual ~TraceLogSink() { TRACE_EVENT_INSTANT1(category_, "vlog", TRACE_EVENT_SCOPE_THREAD, diff --git a/src/kudu/util/flags.cc b/src/kudu/util/flags.cc index 88976894d5..48a9eab725 100644 --- a/src/kudu/util/flags.cc +++ b/src/kudu/util/flags.cc @@ -246,7 +246,7 @@ DECLARE_bool(logtostderr); TAG_FLAG(logtostderr, stable); TAG_FLAG(logtostderr, runtime); -DECLARE_int32(max_log_size); +DECLARE_uint32(max_log_size); TAG_FLAG(max_log_size, stable); TAG_FLAG(max_log_size, runtime); diff --git a/src/kudu/util/logging-test.cc b/src/kudu/util/logging-test.cc index 760cda382d..5514df99d7 100644 --- a/src/kudu/util/logging-test.cc +++ b/src/kudu/util/logging-test.cc @@ -29,6 +29,7 @@ #include #include +#include #include #include "kudu/gutil/strings/substitute.h" @@ -108,7 +109,7 @@ class CountingLogger : public google::base::Logger { void Write(bool force_flush, time_t /*timestamp*/, const char* /*message*/, - int /*message_len*/) override { + size_t /*message_len*/) override { message_count_++; if (force_flush) { Flush(); @@ -222,9 +223,9 @@ TEST(LoggingTest, TestRedactionIllustrateUsage) { ASSERT_EQ("public=abc, private=def", KUDU_DISABLE_REDACTION(SomeComplexStringify("abc", "def"))); // Or we can execute an entire scope with redaction disabled. - KUDU_DISABLE_REDACTION({ + KUDU_DISABLE_REDACTION(({ ASSERT_EQ("public=abc, private=def", SomeComplexStringify("abc", "def")); - }); + })); } diff --git a/src/kudu/util/logging.cc b/src/kudu/util/logging.cc index 3ec98f27e5..fbb41b0249 100644 --- a/src/kudu/util/logging.cc +++ b/src/kudu/util/logging.cc @@ -190,7 +190,7 @@ void FlushCoverageOnExit() { // the first time it's called. // // NOTE: this is only used in coverage builds! -void FailureWriterWithCoverage(const char* data, int size) { +void FailureWriterWithCoverage(const char* data, size_t size) { FlushCoverageOnExit(); // Original implementation from glog: @@ -205,7 +205,7 @@ void FailureWriterWithCoverage(const char* data, int size) { // NOTE: this is only used in coverage builds! void FlushCoverageAndAbort() { FlushCoverageOnExit(); - abort(); + exit(1); } } // anonymous namespace @@ -256,7 +256,8 @@ void InitGoogleLoggingSafe(const char* arg) { // This allows us to handle both LOG(FATAL) and unintended crashes like // SEGVs. google::InstallFailureWriter(FailureWriterWithCoverage); - google::InstallFailureFunction(FlushCoverageAndAbort); + google::InstallFailureFunction( + reinterpret_cast(FlushCoverageAndAbort)); } // Needs to be done after InitGoogleLogging @@ -349,11 +350,11 @@ void GetFullLogFilename(google::LogSeverity severity, string* filename) { std::string FormatTimestampForLog(MicrosecondsInt64 micros_since_epoch) { time_t secs_since_epoch = micros_since_epoch / 1000000; - int usecs = micros_since_epoch % 1000000; + size_t usecs = micros_since_epoch % 1000000; struct tm tm_time; localtime_r(&secs_since_epoch, &tm_time); - return StringPrintf("%02d%02d %02d:%02d:%02d.%06d", + return StringPrintf("%02d%02d %02d:%02d:%02d.%06ld", 1 + tm_time.tm_mon, tm_time.tm_mday, tm_time.tm_hour, @@ -407,7 +408,7 @@ ostream& operator<<(ostream &os, const PRIVATE_ThrottleMsg& /*unused*/) { #endif CHECK(log && log == log->self()) << "You must not use COUNTER with non-glog ostream"; - int ctr = log->ctr(); + size_t ctr = log->ctr(); if (ctr > 0) { os << " [suppressed " << ctr << " similar messages]"; } diff --git a/src/kudu/util/logging.h b/src/kudu/util/logging.h index f8b03b5277..e4bc10d643 100644 --- a/src/kudu/util/logging.h +++ b/src/kudu/util/logging.h @@ -17,6 +17,7 @@ #ifndef KUDU_UTIL_LOGGING_H #define KUDU_UTIL_LOGGING_H +#include #include #include @@ -226,9 +227,6 @@ enum PRIVATE_ThrottleMsg {THROTTLE_MSG}; // The direct user-facing macros. #define KLOG_EVERY_N(severity, n) \ - GOOGLE_GLOG_COMPILE_ASSERT(google::GLOG_ ## severity < \ - google::NUM_SEVERITIES, \ - INVALID_REQUESTED_LOG_SEVERITY); \ KUDU_SOME_KIND_OF_LOG_EVERY_N(severity, (n), google::LogMessage::SendToLog) #define KSYSLOG_EVERY_N(severity, n) \ @@ -327,7 +325,7 @@ class LogThrottler { ANNOTATE_BENIGN_RACE_SIZED(this, sizeof(*this), "OK to be sloppy with log throttling"); } - bool ShouldLog(int n_secs, const char* tag, int* num_suppressed) { + bool ShouldLog(size_t n_secs, const char* tag, int* num_suppressed) { MicrosecondsInt64 ts = GetMonoTimeMicros(); // When we switch tags, we should not show the "suppressed" messages, because diff --git a/src/kudu/util/logging_test_util.h b/src/kudu/util/logging_test_util.h index 8102375085..5b0e568a55 100644 --- a/src/kudu/util/logging_test_util.h +++ b/src/kudu/util/logging_test_util.h @@ -27,9 +27,9 @@ namespace kudu { // GLog sink that keeps an internal buffer of messages that have been logged. class StringVectorSink : public google::LogSink { public: - void send(google::LogSeverity severity, const char* full_filename, + void send(google::LogSeverity severity, const char* /*full_filename*/, const char* base_filename, int line, - const struct ::tm* tm_time, + const google::LogMessageTime& tm_time, const char* message, size_t message_len) override { logged_msgs_.push_back(ToString(severity, base_filename, line, tm_time, message, message_len)); diff --git a/src/kudu/util/minidump.cc b/src/kudu/util/minidump.cc index ef917978c8..66310b4e6a 100644 --- a/src/kudu/util/minidump.cc +++ b/src/kudu/util/minidump.cc @@ -17,6 +17,7 @@ #include "kudu/util/minidump.h" +// IWYU pragma: no_include #include #include @@ -41,6 +42,7 @@ #include #include "kudu/gutil/macros.h" +#include "kudu/gutil/port.h" #include "kudu/gutil/strings/human_readable.h" #include "kudu/util/env.h" #include "kudu/util/env_util.h" @@ -190,7 +192,7 @@ static bool DumpCallback(const google_breakpad::MinidumpDescriptor& descriptor, } // Failure function that simply calls abort(). -static void AbortFailureFunction() { +ATTRIBUTE_NORETURN static void AbortFailureFunction() { abort(); } diff --git a/src/kudu/util/rw_mutex-test.cc b/src/kudu/util/rw_mutex-test.cc index a8254d83d0..34ec8beba3 100644 --- a/src/kudu/util/rw_mutex-test.cc +++ b/src/kudu/util/rw_mutex-test.cc @@ -112,73 +112,73 @@ TEST_P(RWMutexTest, TestDeadlocks) { // Tests that the RWMutex wrapper catches basic usage errors. This checking is // only enabled in debug builds. TEST_P(RWMutexTest, TestLockChecking) { - EXPECT_DEATH({ + EXPECT_DEATH(({ lock_.ReadLock(); lock_.ReadLock(); - }, "already holding lock for reading"); + }), "already holding lock for reading"); - EXPECT_DEATH({ + EXPECT_DEATH(({ CHECK(lock_.TryReadLock()); CHECK(lock_.TryReadLock()); - }, "already holding lock for reading"); + }), "already holding lock for reading"); - EXPECT_DEATH({ + EXPECT_DEATH(({ lock_.ReadLock(); lock_.WriteLock(); - }, "already holding lock for reading"); + }), "already holding lock for reading"); - EXPECT_DEATH({ + EXPECT_DEATH(({ CHECK(lock_.TryReadLock()); CHECK(lock_.TryWriteLock()); - }, "already holding lock for reading"); + }), "already holding lock for reading"); - EXPECT_DEATH({ + EXPECT_DEATH(({ lock_.WriteLock(); lock_.ReadLock(); - }, "already holding lock for writing"); + }), "already holding lock for writing"); - EXPECT_DEATH({ + EXPECT_DEATH(({ CHECK(lock_.TryWriteLock()); CHECK(lock_.TryReadLock()); - }, "already holding lock for writing"); + }), "already holding lock for writing"); - EXPECT_DEATH({ + EXPECT_DEATH(({ lock_.WriteLock(); lock_.WriteLock(); - }, "already holding lock for writing"); + }), "already holding lock for writing"); - EXPECT_DEATH({ + EXPECT_DEATH(({ CHECK(lock_.TryWriteLock()); CHECK(lock_.TryWriteLock()); - }, "already holding lock for writing"); + }), "already holding lock for writing"); - EXPECT_DEATH({ + EXPECT_DEATH(({ lock_.ReadUnlock(); - }, "wasn't holding lock for reading"); + }), "wasn't holding lock for reading"); - EXPECT_DEATH({ + EXPECT_DEATH(({ lock_.WriteUnlock(); - }, "wasn't holding lock for writing"); + }), "wasn't holding lock for writing"); - EXPECT_DEATH({ + EXPECT_DEATH(({ lock_.ReadLock(); lock_.WriteUnlock(); - }, "already holding lock for reading"); + }), "already holding lock for reading"); - EXPECT_DEATH({ + EXPECT_DEATH(({ CHECK(lock_.TryReadLock()); lock_.WriteUnlock(); - }, "already holding lock for reading"); + }), "already holding lock for reading"); - EXPECT_DEATH({ + EXPECT_DEATH(({ lock_.WriteLock(); lock_.ReadUnlock(); - }, "already holding lock for writing"); + }), "already holding lock for writing"); - EXPECT_DEATH({ + EXPECT_DEATH(({ CHECK(lock_.TryWriteLock()); lock_.ReadUnlock(); - }, "already holding lock for writing"); + }), "already holding lock for writing"); } #endif diff --git a/src/kudu/util/sanitizer_options.cc b/src/kudu/util/sanitizer_options.cc index 78b0f37247..54c69344b6 100644 --- a/src/kudu/util/sanitizer_options.cc +++ b/src/kudu/util/sanitizer_options.cc @@ -123,6 +123,13 @@ SANITIZER_HOOK_ATTRIBUTE const char *__tsan_default_suppressions() { // the destruction of a locked mutex in glog. "mutex:glog_internal_namespace_::Mutex::~Mutex\n" + // TODO(martongreber) After doing the LLVM 11 and glog 0.6.0 upgrades the below + // llvm regexes are flagged as race. They are most likely noise. + "race:llvm_regexec\n" + "race:llvm::Regex::match\n" + "race:llvm::Regex::~Regex\n" + "race:llvm_regfree\n" + // gflags variables are accessed without synchronization, but FlagSaver and other // APIs acquire locks when accessing them. This should be safe on x86 for // primitive flag types, but not for string flags, which is why fLS is omitted. diff --git a/thirdparty/build-definitions.sh b/thirdparty/build-definitions.sh index d851e321d7..a4ecfb7e3b 100644 --- a/thirdparty/build-definitions.sh +++ b/thirdparty/build-definitions.sh @@ -196,7 +196,6 @@ build_llvm() { TOOLS_ARGS="$TOOLS_ARGS -DLLVM_TOOL_LIBCXX_BUILD=OFF" TOOLS_ARGS="$TOOLS_ARGS -DLLVM_TOOL_LIBCXXABI_BUILD=OFF" - # Disable some builds we don't care about. for arg in \ CLANG_ENABLE_ARCMT \ @@ -300,14 +299,6 @@ build_llvm() { TOOLS_ARGS="$TOOLS_ARGS -DCOMPILER_RT_ENABLE_TVOS=OFF" fi - # Depend on zlib from the thirdparty tree. It's an optional dependency for - # LLVM, but a required [1] one for IWYU. When TSAN is enabled these flags - # are already set by build-thirdparty.sh in order to support the - # thirdparty libc++, so it's not necessary to set them again. - # - # 1. https://github.com/include-what-you-use/include-what-you-use/issues/539 - CLANG_CXXFLAGS="$CLANG_CXXFLAGS -I$PREFIX/include" - CLANG_LDFLAGS="$CLANG_LDFLAGS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib" ;; "tsan") # Build just the core LLVM libraries, dependent on libc++. @@ -432,32 +423,44 @@ build_libunwind() { } build_glog() { - GLOG_BDIR=$TP_BUILD_DIR/$GLOG_NAME$MODE_SUFFIX - mkdir -p $GLOG_BDIR - pushd $GLOG_BDIR + GLOG_SHARED_BDIR=$TP_BUILD_DIR/$GLOG_NAME.shared$MODE_SUFFIX + GLOG_STATIC_BDIR=$TP_BUILD_DIR/$GLOG_NAME.static$MODE_SUFFIX + for SHARED in ON OFF; do + if [ $SHARED = "ON" ]; then + GLOG_BDIR=$GLOG_SHARED_BDIR + else + GLOG_BDIR=$GLOG_STATIC_BDIR + fi + mkdir -p $GLOG_BDIR + pushd $GLOG_BDIR + rm -rf CMakeCache.txt CMakeFiles/ - # glog depends on libunwind and gflags. - # - # Specifying -Wl,-rpath has different default behavior on GNU binutils ld vs. - # the GNU gold linker. ld sets RPATH (due to defaulting to --disable-new-dtags) - # and gold sets RUNPATH (due to defaulting to --enable-new-dtags). At the time - # of this writing, contrary to the way RPATH is treated, when RUNPATH is - # specified on a binary, glibc doesn't respect it for transitive (non-direct) - # library dependencies (see https://sourceware.org/bugzilla/show_bug.cgi?id=13945). - # So we must set RUNPATH for all deps-of-deps on the dep libraries themselves. - # - # This comment applies both here and the locations elsewhere in this script - # where we add something to -Wl,-rpath. - CXXFLAGS="$EXTRA_CXXFLAGS -I$PREFIX/include" \ - LDFLAGS="$EXTRA_LDFLAGS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib" \ - LIBS="$EXTRA_LIBS" \ - $GLOG_SOURCE/configure \ - --with-pic \ - --prefix=$PREFIX \ - --with-gflags=$PREFIX - fixup_libtool - make -j$PARALLEL $EXTRA_MAKEFLAGS install - popd + # glog depends on libunwind and gflags. + # + # Specifying -Wl,-rpath has different default behavior on GNU binutils ld vs. + # the GNU gold linker. ld sets RPATH (due to defaulting to --disable-new-dtags) + # and gold sets RUNPATH (due to defaulting to --enable-new-dtags). At the time + # of this writing, contrary to the way RPATH is treated, when RUNPATH is + # specified on a binary, glibc doesn't respect it for transitive (non-direct) + # library dependencies (see https://sourceware.org/bugzilla/show_bug.cgi?id=13945). + # So we must set RUNPATH for all deps-of-deps on the dep libraries themselves. + # + # This comment applies both here and the locations elsewhere in this script + # where we add something to -Wl,-rpath. + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DCMAKE_CXX_FLAGS="$EXTRA_CXXFLAGS -I$PREFIX/include" \ + -DCMAKE_EXE_LINKER_FLAGS="$EXTRA_LDFLAGS $EXTRA_LIBS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib" \ + -DCMAKE_MODULE_LINKER_FLAGS="$EXTRA_LDFLAGS $EXTRA_LIBS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib" \ + -DCMAKE_SHARED_LINKER_FLAGS="$EXTRA_LDFLAGS $EXTRA_LIBS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib" \ + -DBUILD_SHARED_LIBS=$SHARED \ + -DBUILD_TESTING=OFF \ + $EXTRA_CMAKE_FLAGS \ + $GLOG_SOURCE + ${NINJA:-make} -j$PARALLEL $EXTRA_MAKEFLAGS install + popd + done } build_gperftools() { diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index b1c7bd4572..0cde4556c3 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -174,11 +174,13 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then # the Kudu build. if ! OPENSSL_CFLAGS=$(pkg-config --cflags openssl); then # If OpenSSL is built via Homebrew, pkg-config does not report on cflags. - homebrew_openssl_dir=/usr/local/opt/openssl - if [ -d $homebrew_openssl_dir ]; then - OPENSSL_CFLAGS="-I$homebrew_openssl_dir/include" - OPENSSL_LDFLAGS="-L$homebrew_openssl_dir/lib" - fi + homebrew_openssl_dirs=(/usr/local/opt/openssl /opt/homebrew/opt/openssl@1.1) + for homebrew_openssl_dir in "${homebrew_openssl_dirs[@]}"; do + if [ -d $homebrew_openssl_dir ]; then + OPENSSL_CFLAGS="-I$homebrew_openssl_dir/include" + OPENSSL_LDFLAGS="-L$homebrew_openssl_dir/lib" + fi + done fi # TSAN doesn't work on macOS. If it was explicitly asked for, respond with an diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index b35aa01f05..e00b8382e4 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -170,22 +170,20 @@ fetch_and_patch() { mkdir -p $TP_SOURCE_DIR cd $TP_SOURCE_DIR -GLOG_PATCHLEVEL=4 +GLOG_PATCHLEVEL=2 fetch_and_patch \ glog-${GLOG_VERSION}.tar.gz \ $GLOG_SOURCE \ $GLOG_PATCHLEVEL \ - "patch -p0 < $TP_DIR/patches/glog-issue-198-fix-unused-warnings.patch" \ - "patch -p0 < $TP_DIR/patches/glog-issue-54-dont-build-tests.patch" \ - "patch -p1 < $TP_DIR/patches/glog-fix-symbolization.patch" \ - "patch -p1 < $TP_DIR/patches/glog-support-stacktrace-for-aarch64.patch" \ - "autoreconf -fvi" + "patch -p1 < $TP_DIR/patches/glog-make-internals-visible.patch" \ + "patch -p1 < $TP_DIR/patches/glog-support-stacktrace-for-aarch64.patch" -GMOCK_PATCHLEVEL=0 +GMOCK_PATCHLEVEL=1 fetch_and_patch \ googletest-release-${GMOCK_VERSION}.tar.gz \ $GMOCK_SOURCE \ - $GMOCK_PATCHLEVEL + $GMOCK_PATCHLEVEL \ + "patch -p1 < $TP_DIR/patches/gmock-remove-unused-gunit-iwyu-pragma.patch" GFLAGS_PATCHLEVEL=0 fetch_and_patch \ @@ -270,12 +268,14 @@ fetch_and_patch \ $LIBEV_PATCHLEVEL \ "patch -p1 < $TP_DIR/patches/libev-c17.patch" -RAPIDJSON_PATCHLEVEL=1 +RAPIDJSON_PATCHLEVEL=3 fetch_and_patch \ rapidjson-${RAPIDJSON_VERSION}.zip \ $RAPIDJSON_SOURCE \ $RAPIDJSON_PATCHLEVEL \ - "patch -p1 < $TP_DIR/patches/rapidjson-fix-signed-unsigned-conversion-error.patch" + "patch -p1 < $TP_DIR/patches/rapidjson-fix-signed-unsigned-conversion-error.patch" \ + "patch -p1 < $TP_DIR/patches/rapidjson-assertions-for-clang-warnings.patch" \ + "patch -p1 < $TP_DIR/patches/rapidjson-avoid-pointer-arithmetic-on-null-pointer.patch" SQUEASEL_PATCHLEVEL=0 fetch_and_patch \ @@ -329,16 +329,12 @@ fetch_and_patch \ $PYTHON_SOURCE \ $PYTHON_PATCHLEVEL -LLVM_PATCHLEVEL=5 +LLVM_PATCHLEVEL=1 fetch_and_patch \ llvm-${LLVM_VERSION}-iwyu-${IWYU_VERSION}.src.tar.gz \ $LLVM_SOURCE \ $LLVM_PATCHLEVEL \ - "patch -p1 < $TP_DIR/patches/llvm-add-iwyu.patch" \ - "patch -p1 < $TP_DIR/patches/llvm-iwyu-include-picker.patch" \ - "patch -p1 < $TP_DIR/patches/llvm-MicrosoftDemangleNodes-e0402b5c9813a2458b8dd3f640883110db280395.patch" \ - "patch -p0 < $TP_DIR/patches/llvm-iwyu-sized-deallocation.patch" \ - "patch -d projects -p1 < $TP_DIR/patches/llvm-947f9692440836dcb8d88b74b69dd379d85974ce.patch" + "patch -p1 < $TP_DIR/patches/llvm-add-iwyu.patch" LZ4_PATCHLEVEL=0 fetch_and_patch \ diff --git a/thirdparty/package-llvm.sh b/thirdparty/package-llvm.sh index ec25174b52..3a71072803 100755 --- a/thirdparty/package-llvm.sh +++ b/thirdparty/package-llvm.sh @@ -36,22 +36,23 @@ # 10. Create new tarball from the resulting source tree # # Usage: -# $ env VERSION=6.0.0 IWYU_VERSION=0.9 thirdparty/package-llvm.sh +# $ env VERSION=11.0.0 IWYU_VERSION=0.15 thirdparty/package-llvm.sh set -eux -for ARTIFACT in llvm cfe compiler-rt libcxx libcxxabi lld clang-tools-extra; do - wget https://releases.llvm.org/$VERSION/$ARTIFACT-$VERSION.src.tar.xz +for ARTIFACT in llvm clang compiler-rt libcxx libcxxabi lld clang-tools-extra; do + wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$VERSION/$ARTIFACT-$VERSION.src.tar.xz tar xf $ARTIFACT-$VERSION.src.tar.xz rm $ARTIFACT-$VERSION.src.tar.xz done IWYU_TAR=include-what-you-use-${IWYU_VERSION}.src.tar.gz wget https://include-what-you-use.org/downloads/$IWYU_TAR -tar xf $IWYU_TAR +mkdir include-what-you-use +tar xf $IWYU_TAR -C include-what-you-use rm $IWYU_TAR -mv cfe-$VERSION.src llvm-$VERSION.src/tools/clang +mv clang-$VERSION.src llvm-$VERSION.src/tools/clang mv clang-tools-extra-$VERSION.src llvm-$VERSION.src/tools/clang/tools/extra mv lld-$VERSION.src llvm-$VERSION.src/tools/lld mv compiler-rt-$VERSION.src llvm-$VERSION.src/projects/compiler-rt diff --git a/thirdparty/patches/glog-fix-symbolization.patch b/thirdparty/patches/glog-fix-symbolization.patch deleted file mode 100644 index 1f11de9af0..0000000000 --- a/thirdparty/patches/glog-fix-symbolization.patch +++ /dev/null @@ -1,212 +0,0 @@ -commit c4d37a7 -Author: Peter Collingbourne -Date: Thu Nov 2 16:53:21 2017 -0700 - - Compute base addresses from program headers while reading /proc/self/maps. - - We previously had logic to compute the base address from program - headers as part of symbolization. The problem is that we need a correct - base address earlier in order to adjust a PC into the image's address - space, as these addresses can appear in unsymbolized output. - - There was previously an assumption that only the mapping that - was lowest in the address space did not need to be adjusted. This - assumption is not guaranteed (for example, the kernel may choose to - map an ET_DYN lowest) and in fact turned out to be wrong in binaries - linked with lld because the first mapping is read-only. - - The solution is to move the program header reading logic into the - code that reads /proc/self/maps. - - There is a change in semantics for clients that install a callback - using the InstallSymbolizeOpenObjectFileCallback function. Any such - clients will need to return a correct base address from the callback - by reading program headers using code similar to that in the function - OpenObjectFileContainingPcAndGetStartAddress. - - (Modified by Adar to remove changes to Makefile.am) - -diff --git a/src/symbolize.cc b/src/symbolize.cc -index 953f1db..98a754f 100644 ---- a/src/symbolize.cc -+++ b/src/symbolize.cc -@@ -56,6 +56,8 @@ - - #if defined(HAVE_SYMBOLIZE) - -+#include -+ - #include - - #include "symbolize.h" -@@ -325,41 +327,17 @@ FindSymbol(uint64_t pc, const int fd, char *out, int out_size, - // both regular and dynamic symbol tables if necessary. On success, - // write the symbol name to "out" and return true. Otherwise, return - // false. --static bool GetSymbolFromObjectFile(const int fd, uint64_t pc, -- char *out, int out_size, -- uint64_t map_base_address) { -+static bool GetSymbolFromObjectFile(const int fd, -+ uint64_t pc, -+ char* out, -+ int out_size, -+ uint64_t base_address) { - // Read the ELF header. - ElfW(Ehdr) elf_header; - if (!ReadFromOffsetExact(fd, &elf_header, sizeof(elf_header), 0)) { - return false; - } - -- uint64_t symbol_offset = 0; -- if (elf_header.e_type == ET_DYN) { // DSO needs offset adjustment. -- ElfW(Phdr) phdr; -- // We need to find the PT_LOAD segment corresponding to the read-execute -- // file mapping in order to correctly perform the offset adjustment. -- for (unsigned i = 0; i != elf_header.e_phnum; ++i) { -- if (!ReadFromOffsetExact(fd, &phdr, sizeof(phdr), -- elf_header.e_phoff + i * sizeof(phdr))) -- return false; -- if (phdr.p_type == PT_LOAD && -- (phdr.p_flags & (PF_R | PF_X)) == (PF_R | PF_X)) { -- // Find the mapped address corresponding to virtual address zero. We do -- // this by first adding p_offset. This gives us the mapped address of -- // the start of the segment, or in other words the mapped address -- // corresponding to the virtual address of the segment. (Note that this -- // is distinct from the start address, as p_offset is not guaranteed to -- // be page aligned.) We then subtract p_vaddr, which takes us to virtual -- // address zero. -- symbol_offset = map_base_address + phdr.p_offset - phdr.p_vaddr; -- break; -- } -- } -- if (symbol_offset == 0) -- return false; -- } -- - ElfW(Shdr) symtab, strtab; - - // Consult a regular symbol table first. -@@ -369,8 +347,7 @@ static bool GetSymbolFromObjectFile(const int fd, uint64_t pc, - symtab.sh_link * sizeof(symtab))) { - return false; - } -- if (FindSymbol(pc, fd, out, out_size, symbol_offset, -- &strtab, &symtab)) { -+ if (FindSymbol(pc, fd, out, out_size, base_address, &strtab, &symtab)) { - return true; // Found the symbol in a regular symbol table. - } - } -@@ -382,8 +359,7 @@ static bool GetSymbolFromObjectFile(const int fd, uint64_t pc, - symtab.sh_link * sizeof(symtab))) { - return false; - } -- if (FindSymbol(pc, fd, out, out_size, symbol_offset, -- &strtab, &symtab)) { -+ if (FindSymbol(pc, fd, out, out_size, base_address, &strtab, &symtab)) { - return true; // Found the symbol in a dynamic symbol table. - } - } -@@ -532,7 +508,6 @@ OpenObjectFileContainingPcAndGetStartAddress(uint64_t pc, - int out_file_name_size) { - int object_fd; - -- // Open /proc/self/maps. - int maps_fd; - NO_INTR(maps_fd = open("/proc/self/maps", O_RDONLY)); - FileDescriptor wrapped_maps_fd(maps_fd); -@@ -540,6 +515,13 @@ OpenObjectFileContainingPcAndGetStartAddress(uint64_t pc, - return -1; - } - -+ int mem_fd; -+ NO_INTR(mem_fd = open("/proc/self/mem", O_RDONLY)); -+ FileDescriptor wrapped_mem_fd(mem_fd); -+ if (wrapped_mem_fd.get() < 0) { -+ return -1; -+ } -+ - // Iterate over maps and look for the map containing the pc. Then - // look into the symbol tables inside. - char buf[1024]; // Big enough for line of sane /proc/self/maps -@@ -575,11 +557,6 @@ OpenObjectFileContainingPcAndGetStartAddress(uint64_t pc, - } - ++cursor; // Skip ' '. - -- // Check start and end addresses. -- if (!(start_address <= pc && pc < end_address)) { -- continue; // We skip this map. PC isn't in this map. -- } -- - // Read flags. Skip flags until we encounter a space or eol. - const char * const flags_start = cursor; - while (cursor < eol && *cursor != ' ') { -@@ -590,6 +567,49 @@ OpenObjectFileContainingPcAndGetStartAddress(uint64_t pc, - return -1; // Malformed line. - } - -+ // Determine the base address by reading ELF headers in process memory. -+ ElfW(Ehdr) ehdr; -+ // Skip non-readable maps. -+ if (flags_start[0] == 'r' && -+ ReadFromOffsetExact(mem_fd, &ehdr, sizeof(ElfW(Ehdr)), start_address) && -+ memcmp(ehdr.e_ident, ELFMAG, SELFMAG) == 0) { -+ switch (ehdr.e_type) { -+ case ET_EXEC: -+ base_address = 0; -+ break; -+ case ET_DYN: -+ // Find the segment containing file offset 0. This will correspond -+ // to the ELF header that we just read. Normally this will have -+ // virtual address 0, but this is not guaranteed. We must subtract -+ // the virtual address from the address where the ELF header was -+ // mapped to get the base address. -+ // -+ // If we fail to find a segment for file offset 0, use the address -+ // of the ELF header as the base address. -+ base_address = start_address; -+ for (unsigned i = 0; i != ehdr.e_phnum; ++i) { -+ ElfW(Phdr) phdr; -+ if (ReadFromOffsetExact( -+ mem_fd, &phdr, sizeof(phdr), -+ start_address + ehdr.e_phoff + i * sizeof(phdr)) && -+ phdr.p_type == PT_LOAD && phdr.p_offset == 0) { -+ base_address = start_address - phdr.p_vaddr; -+ break; -+ } -+ } -+ break; -+ default: -+ // ET_REL or ET_CORE. These aren't directly executable, so they don't -+ // affect the base address. -+ break; -+ } -+ } -+ -+ // Check start and end addresses. -+ if (!(start_address <= pc && pc < end_address)) { -+ continue; // We skip this map. PC isn't in this map. -+ } -+ - // Check flags. We are only interested in "r*x" maps. - if (flags_start[0] != 'r' || flags_start[2] != 'x') { - continue; // We skip this map. -@@ -604,19 +624,6 @@ OpenObjectFileContainingPcAndGetStartAddress(uint64_t pc, - } - ++cursor; // Skip ' '. - -- // Don't subtract 'start_address' from the first entry: -- // * If a binary is compiled w/o -pie, then the first entry in -- // process maps is likely the binary itself (all dynamic libs -- // are mapped higher in address space). For such a binary, -- // instruction offset in binary coincides with the actual -- // instruction address in virtual memory (as code section -- // is mapped to a fixed memory range). -- // * If a binary is compiled with -pie, all the modules are -- // mapped high at address space (in particular, higher than -- // shadow memory of the tool), so the module can't be the -- // first entry. -- base_address = ((num_maps == 1) ? 0U : start_address) - file_offset; -- - // Skip to file name. "cursor" now points to dev. We need to - // skip at least two spaces for dev and inode. - int num_spaces = 0; diff --git a/thirdparty/patches/glog-issue-198-fix-unused-warnings.patch b/thirdparty/patches/glog-issue-198-fix-unused-warnings.patch deleted file mode 100644 index caa8c61e4d..0000000000 --- a/thirdparty/patches/glog-issue-198-fix-unused-warnings.patch +++ /dev/null @@ -1,47 +0,0 @@ -Index: configure.ac -=================================================================== ---- configure.ac (revision 142) -+++ configure.ac (working copy) -@@ -80,15 +80,17 @@ - [Define if you have the 'pwrite' function])) - - AX_C___ATTRIBUTE__ --# We only care about these two attributes. -+# We only care about these attributes. - if test x"$ac_cv___attribute__" = x"yes"; then - ac_cv___attribute___noreturn="__attribute__ ((noreturn))" - ac_cv___attribute___noinline="__attribute__ ((noinline))" - ac_cv___attribute___printf_4_5="__attribute__((__format__ (__printf__, 4, 5)))" -+ ac_cv___attribute___unused="__attribute__ ((unused))" - else - ac_cv___attribute___noreturn= - ac_cv___attribute___noinline= - ac_cv___attribute___printf_4_5= -+ ac_cv___attribute___unused= - fi - - AX_C___BUILTIN_EXPECT -@@ -214,6 +216,7 @@ - AC_SUBST(ac_cv___attribute___noreturn) - AC_SUBST(ac_cv___attribute___noinline) - AC_SUBST(ac_cv___attribute___printf_4_5) -+AC_SUBST(ac_cv___attribute___unused) - AC_SUBST(ac_cv_have___builtin_expect) - AC_SUBST(ac_cv_have_stdint_h) - AC_SUBST(ac_cv_have_systypes_h) -Index: src/glog/logging.h.in -=================================================================== ---- src/glog/logging.h.in (revision 142) -+++ src/glog/logging.h.in (working copy) -@@ -908,8 +908,10 @@ - struct CrashReason; - } // namespace glog_internal_namespace_ - -+#define GOOGLE_GLOG_ATTRIBUTE_UNUSED @ac_cv___attribute___unused@ -+ - #define GOOGLE_GLOG_COMPILE_ASSERT(expr, msg) \ -- typedef @ac_google_namespace@::glog_internal_namespace_::CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1] -+ typedef @ac_google_namespace@::glog_internal_namespace_::CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1] GOOGLE_GLOG_ATTRIBUTE_UNUSED - - #define LOG_EVERY_N(severity, n) \ - GOOGLE_GLOG_COMPILE_ASSERT(@ac_google_namespace@::GLOG_ ## severity < \ diff --git a/thirdparty/patches/glog-issue-54-dont-build-tests.patch b/thirdparty/patches/glog-issue-54-dont-build-tests.patch deleted file mode 100644 index 3d81fabda4..0000000000 --- a/thirdparty/patches/glog-issue-54-dont-build-tests.patch +++ /dev/null @@ -1,147 +0,0 @@ ---- Makefile.am.orig 2016-09-16 13:39:40.053027310 -0700 -+++ Makefile.am 2016-09-16 13:36:27.899844745 -0700 -@@ -52,132 +52,6 @@ - - # The libraries libglog depends on. - COMMON_LIBS = $(PTHREAD_LIBS) $(GFLAGS_LIBS) $(UNWIND_LIBS) --# Compile switches for our unittest. --TEST_CFLAGS = $(GTEST_CFLAGS) $(GMOCK_CFLAGS) $(GFLAGS_CFLAGS) \ -- $(MINGW_CFLAGS) $(AM_CXXFLAGS) --# Libraries for our unittest. --TEST_LIBS = $(GTEST_LIBS) $(GMOCK_LIBS) $(GFLAGS_LIBS) -- --## unittests you want to run when people type 'make check'. --## TESTS is for binary unittests, check_SCRIPTS for script-based unittests. --## TESTS_ENVIRONMENT sets environment variables for when you run unittest, --## but it only seems to take effect for *binary* unittests (argh!) --TESTS = --TESTS_ENVIRONMENT = --check_SCRIPTS = --# Every time you add a unittest to check_SCRIPTS, add it here too --noinst_SCRIPTS = --# Binaries used for script-based unittests. --TEST_BINARIES = -- --TESTS += logging_unittest --logging_unittest_SOURCES = $(gloginclude_HEADERS) \ -- src/logging_unittest.cc \ -- src/config_for_unittests.h \ -- src/mock-log.h --nodist_logging_unittest_SOURCES = $(nodist_gloginclude_HEADERS) --logging_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) --logging_unittest_LDFLAGS = $(PTHREAD_CFLAGS) --logging_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) -- --check_SCRIPTS += logging_striplog_test_sh --noinst_SCRIPTS += src/logging_striplog_test.sh --logging_striplog_test_sh: logging_striptest0 logging_striptest2 logging_striptest10 -- $(top_srcdir)/src/logging_striplog_test.sh -- --check_SCRIPTS += demangle_unittest_sh --noinst_SCRIPTS += src/demangle_unittest.sh --demangle_unittest_sh: demangle_unittest -- $(builddir)/demangle_unittest # force to create lt-demangle_unittest -- $(top_srcdir)/src/demangle_unittest.sh -- --check_SCRIPTS += signalhandler_unittest_sh --noinst_SCRIPTS += src/signalhandler_unittest.sh --signalhandler_unittest_sh: signalhandler_unittest -- $(builddir)/signalhandler_unittest # force to create lt-signalhandler_unittest -- $(top_srcdir)/src/signalhandler_unittest.sh -- --TEST_BINARIES += logging_striptest0 --logging_striptest0_SOURCES = $(gloginclude_HEADERS) \ -- src/logging_striptest_main.cc --nodist_logging_striptest0_SOURCES = $(nodist_gloginclude_HEADERS) --logging_striptest0_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) --logging_striptest0_LDFLAGS = $(PTHREAD_CFLAGS) --logging_striptest0_LDADD = libglog.la $(COMMON_LIBS) -- --TEST_BINARIES += logging_striptest2 --logging_striptest2_SOURCES = $(gloginclude_HEADERS) \ -- src/logging_striptest2.cc --nodist_logging_striptest2_SOURCES = $(nodist_gloginclude_HEADERS) --logging_striptest2_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) --logging_striptest2_LDFLAGS = $(PTHREAD_CFLAGS) --logging_striptest2_LDADD = libglog.la $(COMMON_LIBS) -- --TEST_BINARIES += logging_striptest10 --logging_striptest10_SOURCES = $(gloginclude_HEADERS) \ -- src/logging_striptest10.cc --nodist_logging_striptest10_SOURCES = $(nodist_gloginclude_HEADERS) --logging_striptest10_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) --logging_striptest10_LDFLAGS = $(PTHREAD_CFLAGS) --logging_striptest10_LDADD = libglog.la $(COMMON_LIBS) -- --TESTS += demangle_unittest --demangle_unittest_SOURCES = $(gloginclude_HEADERS) \ -- src/demangle_unittest.cc --nodist_demangle_unittest_SOURCES = $(nodist_gloginclude_HEADERS) --demangle_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) --demangle_unittest_LDFLAGS = $(PTHREAD_CFLAGS) --demangle_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) -- --TESTS += stacktrace_unittest --stacktrace_unittest_SOURCES = $(gloginclude_HEADERS) \ -- src/stacktrace_unittest.cc --nodist_stacktrace_unittest_SOURCES = $(nodist_gloginclude_HEADERS) --stacktrace_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) --stacktrace_unittest_LDFLAGS = $(PTHREAD_CFLAGS) --stacktrace_unittest_LDADD = libglog.la $(COMMON_LIBS) -- --TESTS += symbolize_unittest --symbolize_unittest_SOURCES = $(gloginclude_HEADERS) \ -- src/symbolize_unittest.cc --nodist_symbolize_unittest_SOURCES = $(nodist_gloginclude_HEADERS) --symbolize_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) --symbolize_unittest_LDFLAGS = $(PTHREAD_CFLAGS) --symbolize_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) -- --TESTS += stl_logging_unittest --stl_logging_unittest_SOURCES = $(gloginclude_HEADERS) \ -- src/stl_logging_unittest.cc --nodist_stl_logging_unittest_SOURCES = $(nodist_gloginclude_HEADERS) --stl_logging_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) --stl_logging_unittest_LDFLAGS = $(PTHREAD_CFLAGS) --stl_logging_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) -- --TEST_BINARIES += signalhandler_unittest --signalhandler_unittest_SOURCES = $(gloginclude_HEADERS) \ -- src/signalhandler_unittest.cc --nodist_signalhandler_unittest_SOURCES = $(nodist_gloginclude_HEADERS) --signalhandler_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) --signalhandler_unittest_LDFLAGS = $(PTHREAD_CFLAGS) --signalhandler_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) -- --TESTS += utilities_unittest --utilities_unittest_SOURCES = $(gloginclude_HEADERS) \ -- src/utilities_unittest.cc --nodist_utilities_unittest_SOURCES = $(nodist_gloginclude_HEADERS) --utilities_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) --utilities_unittest_LDFLAGS = $(PTHREAD_CFLAGS) --utilities_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) -- --if HAVE_GMOCK --TESTS += mock_log_test --mock_log_test_SOURCES = $(gloginclude_HEADERS) \ -- src/mock-log_test.cc --nodist_mock_log_test_SOURCES = $(nodist_gloginclude_HEADERS) --mock_log_test_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) --mock_log_test_LDFLAGS = $(PTHREAD_CFLAGS) --mock_log_test_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) --endif - - ## vvvv RULES TO MAKE THE LIBRARIES, BINARIES, AND UNITTESTS - -@@ -212,11 +86,6 @@ - - ## ^^^^ END OF RULES TO MAKE THE LIBRARIES, BINARIES, AND UNITTESTS - -- --## This should always include $(TESTS), but may also include other --## binaries that you compile but don't want automatically installed. --noinst_PROGRAMS = $(TESTS) $(TEST_BINARIES) -- - rpm: dist-gzip packages/rpm.sh packages/rpm/rpm.spec - @cd packages && ./rpm.sh ${PACKAGE} ${VERSION} - diff --git a/thirdparty/patches/glog-make-internals-visible.patch b/thirdparty/patches/glog-make-internals-visible.patch new file mode 100644 index 0000000000..484fe67db1 --- /dev/null +++ b/thirdparty/patches/glog-make-internals-visible.patch @@ -0,0 +1,23 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 846b444..5c5d42e 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -60,10 +60,7 @@ if (NOT WITH_THREADS) + set (CMAKE_DISABLE_FIND_PACKAGE_Threads ON) + endif (NOT WITH_THREADS) + +-set (CMAKE_C_VISIBILITY_PRESET hidden) +-set (CMAKE_CXX_VISIBILITY_PRESET hidden) + set (CMAKE_POSITION_INDEPENDENT_CODE ON) +-set (CMAKE_VISIBILITY_INLINES_HIDDEN ON) + + set (CMAKE_DEBUG_POSTFIX d) + set (CMAKE_THREAD_PREFER_PTHREAD 1) +@@ -581,6 +578,7 @@ set (GLOG_PUBLIC_H + ${CMAKE_CURRENT_BINARY_DIR}/glog/vlog_is_on.h + src/glog/log_severity.h + src/glog/platform.h ++ src/utilities.h + ) + + set (GLOG_SRCS diff --git a/thirdparty/patches/glog-support-stacktrace-for-aarch64.patch b/thirdparty/patches/glog-support-stacktrace-for-aarch64.patch index 303d3a70e9..78346571b4 100644 --- a/thirdparty/patches/glog-support-stacktrace-for-aarch64.patch +++ b/thirdparty/patches/glog-support-stacktrace-for-aarch64.patch @@ -110,41 +110,14 @@ index 0000000..1e3ce62 + +_END_GOOGLE_NAMESPACE_ - -diff --git a/Makefile.am b/Makefile.am -index 0c87c89..886594c 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -64,6 +64,7 @@ libglog_la_SOURCES = $(gloginclude_HEADERS) \ - src/stacktrace_generic-inl.h \ - src/stacktrace_libunwind-inl.h \ - src/stacktrace_powerpc-inl.h \ -+ src/stacktrace_aarch64-inl.h \ - src/stacktrace_x86-inl.h \ - src/stacktrace_x86_64-inl.h \ - src/symbolize.cc src/symbolize.h \ - diff --git a/src/utilities.h b/src/utilities.h index 5f79968..3ca051b 100644 --- a/src/utilities.h +++ b/src/utilities.h -@@ -102,7 +102,11 @@ - // Some code may do that. - - #if defined(HAVE_LIB_UNWIND) --# define STACKTRACE_H "stacktrace_libunwind-inl.h" -+# if defined(__aarch64__) -+# define STACKTRACE_H "stacktrace_aarch64-inl.h" -+# else -+# define STACKTRACE_H "stacktrace_libunwind-inl.h" -+# endif - #elif !defined(NO_FRAME_POINTER) - # if defined(__i386__) && __GNUC__ >= 2 - # define STACKTRACE_H "stacktrace_x86-inl.h" -@@ -110,6 +114,8 @@ - # define STACKTRACE_H "stacktrace_x86_64-inl.h" - # elif (defined(__ppc__) || defined(__PPC__)) && __GNUC__ >= 2 +@@ -97,6 +97,8 @@ # define STACKTRACE_H "stacktrace_powerpc-inl.h" + # elif defined(GLOG_OS_WINDOWS) + # define STACKTRACE_H "stacktrace_windows-inl.h" +# elif defined(__aarch64__) +# define STACKTRACE_H "stacktrace_aarch64-inl.h" # endif diff --git a/thirdparty/patches/gmock-remove-unused-gunit-iwyu-pragma.patch b/thirdparty/patches/gmock-remove-unused-gunit-iwyu-pragma.patch new file mode 100644 index 0000000000..9077631139 --- /dev/null +++ b/thirdparty/patches/gmock-remove-unused-gunit-iwyu-pragma.patch @@ -0,0 +1,11 @@ +diff --git a/googletest/include/gtest/gtest-matchers.h b/googletest/include/gtest/gtest-matchers.h +index 9de6c2e1..f9ef74ef 100644 +--- a/googletest/include/gtest/gtest-matchers.h ++++ b/googletest/include/gtest/gtest-matchers.h +@@ -32,7 +32,6 @@ + // This file implements just enough of the matcher interface to allow + // EXPECT_DEATH and friends to accept a matcher argument. + +-// IWYU pragma: private, include "testing/base/public/gunit.h" + // IWYU pragma: friend third_party/googletest/googlemock/.* + // IWYU pragma: friend third_party/googletest/googletest/.* diff --git a/thirdparty/patches/llvm-iwyu-include-picker.patch b/thirdparty/patches/llvm-iwyu-include-picker.patch deleted file mode 100644 index 5c42d15a2a..0000000000 --- a/thirdparty/patches/llvm-iwyu-include-picker.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- a/tools/clang/tools/include-what-you-use/iwyu_include_picker.cc 2017-08-07 16:41:03.193267222 -0700 -+++ b/tools/clang/tools/include-what-you-use/iwyu_include_picker.cc 2017-08-07 16:41:12.603327180 -0700 -@@ -288,6 +295,7 @@ const IncludeMapEntry libc_include_map[] = { - { "", kPrivate, "", kPublic }, - { "", kPrivate, "", kPrivate }, - { "", kPrivate, "", kPrivate }, -+ { "", kPrivate, "", kPublic }, - { "", kPrivate, "", kPublic }, - { "", kPrivate, "", kPublic }, - { "", kPrivate, "", kPublic }, diff --git a/thirdparty/patches/llvm-iwyu-sized-deallocation.patch b/thirdparty/patches/llvm-iwyu-sized-deallocation.patch deleted file mode 100644 index 5af6398c2c..0000000000 --- a/thirdparty/patches/llvm-iwyu-sized-deallocation.patch +++ /dev/null @@ -1,116 +0,0 @@ ---- tools/clang/tools/include-what-you-use/iwyu_ast_util.cc.orig 2020-03-23 14:03:01.060932783 -0700 -+++ tools/clang/tools/include-what-you-use/iwyu_ast_util.cc 2020-03-23 14:04:37.056235116 -0700 -@@ -47,6 +47,7 @@ - class FileEntry; - } // namespace clang - -+using clang::ASTContext; - using clang::BlockPointerType; - using clang::CXXConstructExpr; - using clang::CXXConstructorDecl; -@@ -78,6 +79,7 @@ - using clang::FullSourceLoc; - using clang::FunctionDecl; - using clang::FunctionType; -+using clang::IdentifierInfo; - using clang::ImplicitCastExpr; - using clang::InjectedClassNameType; - using clang::LValueReferenceType; -@@ -929,13 +931,81 @@ - !StartsWith(decl_name, "operator delete")) - return false; - -- // Placement-new/delete has 2 args, second is void*. The only other -- // 2-arg overloads of new/delete in take a const nothrow_t&. -- if (decl->getNumParams() == 2 && -- !decl->getParamDecl(1)->getType().isConstQualified()) -- return false; -- -- return true; -+ // The following variants of operator new[1] are implicitly defined in every -+ // translation unit and should not require including . -+ // -+ // void* operator new ( std::size_t count ); -+ // void* operator new[]( std::size_t count ); -+ // void* operator new ( std::size_t count, std::align_val_t al ); (since C++17) -+ // void* operator new[]( std::size_t count, std::align_val_t al ); (since C++17) -+ // -+ // Likewise, the following variants of operator delete[2] are implicitly -+ // defined in every translation unit and should not require including . -+ // -+ // void operator delete ( void* ptr ) throw(); (until C++11) -+ // void operator delete ( void* ptr ) noexcept; (since C++11) -+ // void operator delete[]( void* ptr ) throw(); (until C++11) -+ // void operator delete[]( void* ptr ) noexcept; (since C++11) -+ // void operator delete ( void* ptr, std::align_val_t al ) noexcept; (since C++17) -+ // void operator delete[]( void* ptr, std::align_val_t al ) noexcept; (since C++17) -+ // void operator delete ( void* ptr, std::size_t sz ) noexcept; (since C++14) -+ // void operator delete[]( void* ptr, std::size_t sz ) noexcept; (since C++14) -+ // void operator delete ( void* ptr, std::size_t sz, -+ // std::align_val_t al ) noexcept; (since C++17) -+ // void operator delete[]( void* ptr, std::size_t sz, -+ // std::align_val_t al ) noexcept; (since C++17) -+ // void operator delete ( void* ptr, const std::nothrow_t& tag ) throw(); (until C++11) -+ // void operator delete ( void* ptr, const std::nothrow_t& tag ) noexcept; (since C++11) -+ // void operator delete[]( void* ptr, const std::nothrow_t& tag ) throw(); (until C++11) -+ // void operator delete[]( void* ptr, const std::nothrow_t& tag ) noexcept; (since C++11) -+ // -+ // The below code attempts to return true for these variants while returning -+ // false for all others. FunctionDecl::isReplaceableGlobalAllocationFunction -+ // comes very very close, but returns true for nothrow new, which is not -+ // implicitly defined. -+ // -+ // 1. https://en.cppreference.com/w/cpp/memory/new/operator_new -+ // 2. https://en.cppreference.com/w/cpp/memory/new/operator_delete -+ switch (decl->getNumParams()) { -+ case 1: -+ // All 1-arg variants are implicitly declared. -+ return true; -+ case 2: { -+ // Amongst 2-arg variants, aligned (C++17) new/delete, sized delete (C++14), and -+ // nothrow delete are implicitly declared. -+ ASTContext& ctx = decl->getASTContext(); -+ QualType t = decl->getParamDecl(1)->getType(); -+ if (t->isAlignValT() || // aligned new/delete -+ ctx.hasSameType(t, ctx.getSizeType())) // sized delete -+ return true; -+ // We have to work a bit harder to figure out if it's a nothrow delete. -+ // -+ // This cribs from FunctionDecl::isReplaceableGlobalAllocationFunction. -+ if (StartsWith(decl_name, "operator delete") && t->isReferenceType()) { -+ t = t->getPointeeType(); -+ if (t.isConstQualified()) { -+ const CXXRecordDecl* recordDecl = t->getAsCXXRecordDecl(); -+ if (recordDecl) { -+ const IdentifierInfo* iInfo = recordDecl->getIdentifier(); -+ if (iInfo && iInfo->isStr("nothrow_t") && recordDecl->isInStdNamespace()) -+ return true; -+ } -+ } -+ } -+ return false; -+ } -+ case 3: { -+ // Amongst 3-arg variants, only sized aligned delete (C++17) is implicitly -+ // declared. -+ ASTContext& ctx = decl->getASTContext(); -+ QualType t = decl->getParamDecl(1)->getType(); -+ return ctx.hasSameType(t, ctx.getSizeType()) && -+ decl->getParamDecl(2)->getType()->isAlignValT(); -+ } -+ default: -+ return false; -+ return true; -+ } - } - - bool IsFriendDecl(const Decl* decl) { -@@ -1082,7 +1152,7 @@ - - bool IsBuiltinFunction(const clang::NamedDecl* decl, - const std::string& symbol_name) { -- if (const clang::IdentifierInfo* iden = decl->getIdentifier()) { -+ if (const IdentifierInfo* iden = decl->getIdentifier()) { - return iden->getBuiltinID() != 0 && - !clang::Builtin::Context::isBuiltinFunc(symbol_name.c_str()); - } diff --git a/thirdparty/patches/rapidjson-assertions-for-clang-warnings.patch b/thirdparty/patches/rapidjson-assertions-for-clang-warnings.patch new file mode 100644 index 0000000000..1f9a74d777 --- /dev/null +++ b/thirdparty/patches/rapidjson-assertions-for-clang-warnings.patch @@ -0,0 +1,21 @@ +From 9d8df28c1dd92be8480fae8026fed0aa2c0d8cdd Mon Sep 17 00:00:00 2001 +From: Patrick Cheng +Date: Fri, 30 Sep 2016 10:47:00 -0700 +Subject: [PATCH] added assertion to help suppress clang warnings + +--- + include/rapidjson/internal/stack.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/include/rapidjson/internal/stack.h b/include/rapidjson/internal/stack.h +index 022c9aab4..54ac77a82 100644 +--- a/include/rapidjson/internal/stack.h ++++ b/include/rapidjson/internal/stack.h +@@ -126,6 +127,7 @@ class Stack { + + template + RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { ++ RAPIDJSON_ASSERT(stackTop_); + RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); + T* ret = reinterpret_cast(stackTop_); + stackTop_ += sizeof(T) * count; diff --git a/thirdparty/patches/rapidjson-avoid-pointer-arithmetic-on-null-pointer.patch b/thirdparty/patches/rapidjson-avoid-pointer-arithmetic-on-null-pointer.patch new file mode 100644 index 0000000000..5aae0af832 --- /dev/null +++ b/thirdparty/patches/rapidjson-avoid-pointer-arithmetic-on-null-pointer.patch @@ -0,0 +1,43 @@ +From 16872af88915176f49e389defb167f899e2c230a Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Konstantin=20K=C3=A4fer?= +Date: Thu, 1 Sep 2016 12:10:03 +0200 +Subject: [PATCH] Avoid pointer arithmetic on null pointer to remove undefined + behavior + +The existing checks triggered undefined behavior when the stack was empty (null pointer). This change avoid this: +* If `stackTop_` and `stackEnd_` are null, it results in a `ptrdiff_t` of `0` +* If `stackTop_` and `stackEnd_` are valid pointers, they produce a `ptrdiff_t` with the remaining size on the stack +--- + include/rapidjson/internal/stack.h | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/include/rapidjson/internal/stack.h b/include/rapidjson/internal/stack.h +index 89558d0da..45dca6a8b 100644 +--- a/include/rapidjson/internal/stack.h ++++ b/include/rapidjson/internal/stack.h +@@ -17,6 +17,7 @@ + + #include "../allocators.h" + #include "swap.h" ++#include + + #if defined(__clang__) + RAPIDJSON_DIAG_PUSH +@@ -114,7 +115,7 @@ class Stack { + template + RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { + // Expand the stack if needed +- if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) ++ if (RAPIDJSON_UNLIKELY(static_cast(sizeof(T) * count) > (stackEnd_ - stackTop_))) + Expand(count); + } + +@@ -127,7 +128,7 @@ class Stack { + template + RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { + RAPIDJSON_ASSERT(stackTop_); +- RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); ++ RAPIDJSON_ASSERT(static_cast(sizeof(T) * count) <= (stackEnd_ - stackTop_)); + T* ret = reinterpret_cast(stackTop_); + stackTop_ += sizeof(T) * count; + return ret; diff --git a/thirdparty/preflight.py b/thirdparty/preflight.py index f8fc0ddb53..b75ef453ec 100755 --- a/thirdparty/preflight.py +++ b/thirdparty/preflight.py @@ -138,9 +138,10 @@ def check_openssl(): if openssl_include != "": cflags.append(openssl_include) else: - homebrew_openssl_dir="/usr/local/opt/openssl/include" - if os.path.isdir(homebrew_openssl_dir): - cflags.append("-I" + homebrew_openssl_dir) + homebrew_openssl_dirs=["/usr/local/opt/openssl/include", "/opt/homebrew/opt/openssl@1.1/include"] + for homebrew_openssl_dir in homebrew_openssl_dirs: + if os.path.isdir(homebrew_openssl_dir): + cflags.append("-I" + homebrew_openssl_dir) try_do( "Checking for openssl headers", ("Unable to compile a simple program that uses openssl. " + diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 4e94419f3a..55f16421b3 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -42,7 +42,7 @@ GFLAGS_VERSION=2.2.2 GFLAGS_NAME=gflags-$GFLAGS_VERSION GFLAGS_SOURCE=$TP_SOURCE_DIR/$GFLAGS_NAME -GLOG_VERSION=0.3.5 +GLOG_VERSION=0.6.0 GLOG_NAME=glog-$GLOG_VERSION GLOG_SOURCE=$TP_SOURCE_DIR/$GLOG_NAME @@ -142,12 +142,12 @@ LIBUNWIND_NAME=libunwind-$LIBUNWIND_VERSION LIBUNWIND_SOURCE=$TP_SOURCE_DIR/$LIBUNWIND_NAME # See package-llvm.sh for details on the LLVM tarball. -LLVM_VERSION=9.0.0 +LLVM_VERSION=11.0.0 LLVM_NAME=llvm-$LLVM_VERSION.src LLVM_SOURCE=$TP_SOURCE_DIR/$LLVM_NAME # The include-what-you-use is built along with LLVM in its source tree. -IWYU_VERSION=0.13 +IWYU_VERSION=0.15 # Python is required to build LLVM 3.6+ because it uses # llvm/utils/llvm-build/llvmbuild script. It is only built and installed if