From 2cf8c65b5e696bc1be4df7401830c12462c0a0e4 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 14 May 2024 16:25:19 +0000 Subject: [PATCH 01/80] WIP --- ddprof-lib/build.gradle | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 2101e42ec..677c0dd00 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -67,10 +67,18 @@ tasks.withType(CppCompile).configureEach { if (it.name.contains('Debug')) { taskArgs.add("-DDEBUG") + + if (os().isLinux()) { + taskArgs.add "-g" + taskArgs.add "-fsanitize=address" + taskArgs.add "-fno-sanitize-recover=all" + } } else { taskArgs.add("-DNDEBUG") } + println "Compiler args: ${taskArgs}" + // TODO for Clang we would use scan-build utility instead but it needs to be installed first taskIncludes.add "${projectDir}/../malloc-shim/src/main/public" @@ -88,12 +96,18 @@ tasks.withType(LinkSharedLibrary) { onlyIf { !project.hasProperty('skip-native') } - linkerArgs.addAll(["-ldl", "-lpthread", "-lm"]) + def taskArgs = ["-ldl", "-lpthread", "-lm", "-v"] if (os().isLinux()) { - linkerArgs.addAll(["-Wl,-z,defs", "-Wl,-z,nodelete", "-lrt", "-static-libstdc++", "-static-libgcc", - "-Wl,--exclude-libs,ALL", "-Wl,--gc-sections"]) + if (it.name.contains('Debug')) { + println "Adding address sanitizer" + taskArgs.add "-fsanitize=address" + } else { + taskArgs.addAll(["-Wl,-z,defs", "-Wl,-z,nodelete", "-lrt", "-static-libstdc++", "-static-libgcc", + "-Wl,--exclude-libs,ALL", "-Wl,--gc-sections"]) + } } - linkerArgs.add '-v' + println "Linker args: ${taskArgs}" + linkerArgs.addAll(taskArgs) } tasks.withType(StripSymbols) { From c9de9da322460c7cdf17a39087ae8ef7778c976e Mon Sep 17 00:00:00 2001 From: r1viollet Date: Wed, 15 May 2024 11:06:14 +0200 Subject: [PATCH 02/80] Work in progress to try out sanitizers --- ddprof-lib/build.gradle | 41 +++++++++++++++++++++++++++++----------- ddprof-test/build.gradle | 3 +++ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 677c0dd00..99f2ef8f6 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -7,13 +7,13 @@ plugins { id 'com.github.ben-manes.versions' version '0.27.0' id "com.diffplug.spotless" version "6.11.0" } - def libraryName = "ddprof" description = "Datadog Java Profiler Library" def component_version = project.hasProperty("ddprof_version") ? project.ddprof_version : project.version + // this feels weird but it is the only way invoking `./gradlew :ddprof-lib:*` tasks will work if (rootDir.toString().endsWith("ddprof-lib")) { apply from: rootProject.file('../common.gradle') @@ -68,11 +68,18 @@ tasks.withType(CppCompile).configureEach { if (it.name.contains('Debug')) { taskArgs.add("-DDEBUG") - if (os().isLinux()) { - taskArgs.add "-g" - taskArgs.add "-fsanitize=address" - taskArgs.add "-fno-sanitize-recover=all" - } +// /usr/bin/c++ -fsanitize=undefined -fsanitize=float-divide-by-zero -fno-sanitize-recover -fsanitize=address -fstack-protector-all -std=gnu++20 -MD -MT CMakeFiles/test.dir/main.o -MF CMakeFiles/test.dir/main.o.d -o CMakeFiles/test.dir/main.o -c /home/r1viollet/dd/system-experiments/segvs/main.cpp + + if (os().isLinux()) { + taskArgs.add "-g" + taskArgs.add "-fsanitize=address" + // taskArgs.add "-static-libasan" + // taskArgs.add "-fno-sanitize-recover=all" + // taskArgs.add "-fsanitize=undefined" + // taskArgs.add "-fsanitize=float-divide-by-zero" + // taskArgs.add "-fstack-protector-all" + + } } else { taskArgs.add("-DNDEBUG") } @@ -96,14 +103,26 @@ tasks.withType(LinkSharedLibrary) { onlyIf { !project.hasProperty('skip-native') } - def taskArgs = ["-ldl", "-lpthread", "-lm", "-v"] + def taskArgs = [] if (os().isLinux()) { + taskArgs.addAll(["-ldl", "-Wl,-z,defs", "--verbose", "-lpthread", "-lm", "-v"]) // Common flags if (it.name.contains('Debug')) { - println "Adding address sanitizer" - taskArgs.add "-fsanitize=address" + println "Adding debug-specific linker args" + taskArgs.addAll([ + "-fsanitize=address", + // "-static-libasan", // Ensure the ASan runtime is linked statically + "-fno-omit-frame-pointer" // Often recommended with sanitizers + ]) } else { - taskArgs.addAll(["-Wl,-z,defs", "-Wl,-z,nodelete", "-lrt", "-static-libstdc++", "-static-libgcc", - "-Wl,--exclude-libs,ALL", "-Wl,--gc-sections"]) + println "Adding release-specific linker args" + taskArgs.addAll([ + "-Wl,-z,nodelete", + "-lrt", + "-static-libstdc++", + "-static-libgcc", + "-Wl,--exclude-libs,ALL", + "-Wl,--gc-sections" + ]) } } println "Linker args: ${taskArgs}" diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index c55989133..680a8d049 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -37,6 +37,9 @@ test { } tasks.withType(Test) { + environment "LD_PRELOAD", "/usr/lib/gcc/x86_64-linux-gnu/11/libasan.so" + environment "ASAN_OPTIONS", "detect_leaks=0" + onlyIf { !project.hasProperty('skip-tests') } From febacfb56b113e3a357ee15c18df8afb1c467561 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Wed, 15 May 2024 14:10:08 +0200 Subject: [PATCH 03/80] Add suppression options --- ddprof-lib/build.gradle | 12 ++++++++---- ddprof-lib/src/main/cpp/buffers.h | 4 ++++ ddprof-lib/src/main/cpp/dwarf.h | 8 ++++++-- ddprof-test/asan.supp | 1 + ddprof-test/build.gradle | 4 ++-- .../test/cmake/asan.supp => ddprof-test/ubsan.supp | 0 6 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 ddprof-test/asan.supp rename ddprof-lib/src/test/cmake/asan.supp => ddprof-test/ubsan.supp (100%) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 99f2ef8f6..1487c346c 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -73,11 +73,14 @@ tasks.withType(CppCompile).configureEach { if (os().isLinux()) { taskArgs.add "-g" taskArgs.add "-fsanitize=address" + taskArgs.add "-fsanitize=undefined" + // static could be interesting // taskArgs.add "-static-libasan" - // taskArgs.add "-fno-sanitize-recover=all" - // taskArgs.add "-fsanitize=undefined" - // taskArgs.add "-fsanitize=float-divide-by-zero" - // taskArgs.add "-fstack-protector-all" + // This one we should test + // -fsanitize-address-use-after-return=always + taskArgs.add "-fno-sanitize-recover=all" + taskArgs.add "-fsanitize=float-divide-by-zero" + taskArgs.add "-fstack-protector-all" } } else { @@ -110,6 +113,7 @@ tasks.withType(LinkSharedLibrary) { println "Adding debug-specific linker args" taskArgs.addAll([ "-fsanitize=address", + "-fsanitize=undefined", // "-static-libasan", // Ensure the ASan runtime is linked statically "-fno-omit-frame-pointer" // Often recommended with sanitizers ]) diff --git a/ddprof-lib/src/main/cpp/buffers.h b/ddprof-lib/src/main/cpp/buffers.h index 0c0c11cfd..e221fda10 100644 --- a/ddprof-lib/src/main/cpp/buffers.h +++ b/ddprof-lib/src/main/cpp/buffers.h @@ -93,6 +93,10 @@ class Buffer { _offset += 4; } + // alignment issue to be looked at + // runtime error: store to misaligned address 0x766bb5a1e814 for type 'u64', which requires 8 byte alignment + // 0x766bb5a1e814: note: pointer points here + __attribute__((no_sanitize("undefined"))) void put64(u64 v) { assert(_offset + 8 < limit()); *(u64*)(_data + _offset) = OS::hton64(v); diff --git a/ddprof-lib/src/main/cpp/dwarf.h b/ddprof-lib/src/main/cpp/dwarf.h index 0168cd04f..72788d7c2 100644 --- a/ddprof-lib/src/main/cpp/dwarf.h +++ b/ddprof-lib/src/main/cpp/dwarf.h @@ -112,11 +112,15 @@ class DwarfParser { u8 get8() { return *_ptr++; } - + // We are getting alignment issues when loading the 16-bit value + // todo: are these relevant and well handled ? + __attribute__((no_sanitize("undefined"))) u16 get16() { return *(u16*)add(2); } + // same issue + __attribute__((no_sanitize("undefined"))) u32 get32() { return *(u32*)add(4); } @@ -139,7 +143,7 @@ class DwarfParser { result |= (b & 0x7f) << shift; if ((b & 0x80) == 0) { if ((b & 0x40) != 0 && (shift += 7) < 32) { - result |= -1 << shift; + result |= (static_cast(-1)) << shift; } return result; } diff --git a/ddprof-test/asan.supp b/ddprof-test/asan.supp new file mode 100644 index 000000000..7041f6237 --- /dev/null +++ b/ddprof-test/asan.supp @@ -0,0 +1 @@ +interceptor_via_fun:get16 diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index 680a8d049..3f7fa11d4 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -38,8 +38,8 @@ test { tasks.withType(Test) { environment "LD_PRELOAD", "/usr/lib/gcc/x86_64-linux-gnu/11/libasan.so" - environment "ASAN_OPTIONS", "detect_leaks=0" - + environment "ASAN_OPTIONS", "detect_leaks=0,suppressions=${projectDir}/asan.supp" + environment "UBSAN_OPTIONS", "detect_leaks=0,suppressions=${projectDir}/ubsan.supp" onlyIf { !project.hasProperty('skip-tests') } diff --git a/ddprof-lib/src/test/cmake/asan.supp b/ddprof-test/ubsan.supp similarity index 100% rename from ddprof-lib/src/test/cmake/asan.supp rename to ddprof-test/ubsan.supp From d26011e306bd7ed1ce16056d4402894d539ffccb Mon Sep 17 00:00:00 2001 From: r1viollet Date: Wed, 15 May 2024 14:58:44 +0200 Subject: [PATCH 04/80] Avoid division by 0 --- ddprof-lib/src/main/cpp/livenessTracker.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/src/main/cpp/livenessTracker.cpp b/ddprof-lib/src/main/cpp/livenessTracker.cpp index c721b7727..f01ef195a 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.cpp +++ b/ddprof-lib/src/main/cpp/livenessTracker.cpp @@ -138,8 +138,10 @@ void LivenessTracker::flush_table(std::set *tracked_thread_ids) { } end = OS::nanotime(); - Log::debug("Liveness tracker flush took %.2fms (%.2fus/element)", - 1.0f * (end - start) / 1000 / 1000, 1.0f * (end - start) / 1000 / sz); + if (sz) { + Log::debug("Liveness tracker flush took %.2fms (%.2fus/element)", + 1.0f * (end - start) / 1000 / 1000, 1.0f * (end - start) / 1000 / sz); + } } Error LivenessTracker::initialize_table(JNIEnv* jni, int sampling_interval) { From 7435234caf55786bc929a2b10f761ca64911b4b7 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Wed, 22 May 2024 16:11:49 +0200 Subject: [PATCH 05/80] Tsan fix - Linear allocator Add atomic load --- ddprof-lib/src/main/cpp/linearAllocator.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ddprof-lib/src/main/cpp/linearAllocator.cpp b/ddprof-lib/src/main/cpp/linearAllocator.cpp index 484bd9cfe..7d3ae0df3 100644 --- a/ddprof-lib/src/main/cpp/linearAllocator.cpp +++ b/ddprof-lib/src/main/cpp/linearAllocator.cpp @@ -43,11 +43,11 @@ void LinearAllocator::clear() { } void* LinearAllocator::alloc(size_t size) { - Chunk* chunk = _tail; - + Chunk* chunk = __atomic_load_n(&_tail, __ATOMIC_ACQUIRE); do { // Fast path: bump a pointer with CAS - for (size_t offs = chunk->offs; offs + size <= _chunk_size; offs = chunk->offs) { + for (size_t offs = __atomic_load_n(&chunk->offs, __ATOMIC_ACQUIRE); offs + size <= _chunk_size; + offs = __atomic_load_n(&chunk->offs, __ATOMIC_ACQUIRE)) { if (__sync_bool_compare_and_swap(&chunk->offs, offs, offs + size)) { if (_chunk_size / 2 - offs < size) { // Stepped over a middle of the chunk - it's time to prepare a new one From ea6aa492b2c68ab023b5369a7fce8f473fac8ca5 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Wed, 22 May 2024 16:13:18 +0200 Subject: [PATCH 06/80] stackFrame - minor asan fix Ensure we have an aligned memory access to remove the asan warning --- ddprof-lib/src/main/cpp/stackFrame_x64.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/stackFrame_x64.cpp b/ddprof-lib/src/main/cpp/stackFrame_x64.cpp index 4a406dd8f..64e06db13 100644 --- a/ddprof-lib/src/main/cpp/stackFrame_x64.cpp +++ b/ddprof-lib/src/main/cpp/stackFrame_x64.cpp @@ -86,6 +86,7 @@ void StackFrame::ret() { bool StackFrame::unwindStub(instruction_t* entry, const char* name, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp) { + instruction_t* ip = (instruction_t*)pc; if (ip == entry || *ip == 0xc3 || strncmp(name, "itable", 6) == 0 @@ -95,7 +96,14 @@ bool StackFrame::unwindStub(instruction_t* entry, const char* name, uintptr_t& p pc = ((uintptr_t*)sp)[0] - 1; sp += 8; return true; - } else if (entry != NULL && *(unsigned int*)entry == 0xec8b4855) { + } + if(entry == nullptr) { + return false; + } + if (reinterpret_cast(entry) % alignof(unsigned int) != 0) { + return false; + } + if (*(unsigned int*)entry == 0xec8b4855) { // The stub begins with // push rbp // mov rbp, rsp From 336384d1a2ae28e2de80ae563bba73a5557cfc25 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Thu, 23 May 2024 15:51:04 +0200 Subject: [PATCH 07/80] Tsan spinlock fix Ensure that we load the lock value in the loop --- ddprof-lib/src/main/cpp/spinLock.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/src/main/cpp/spinLock.h b/ddprof-lib/src/main/cpp/spinLock.h index 63e27e042..b472ed8f7 100644 --- a/ddprof-lib/src/main/cpp/spinLock.h +++ b/ddprof-lib/src/main/cpp/spinLock.h @@ -51,9 +51,9 @@ class SpinLock { __sync_fetch_and_sub(&_lock, 1); } - bool tryLockShared() { + bool SpinLock::tryLockShared() { int value; - while ((value = _lock) <= 0) { + while ((value = __atomic_load_n(&_lock, __ATOMIC_ACQUIRE)) <= 0) { if (__sync_bool_compare_and_swap(&_lock, value, value - 1)) { return true; } From 1ed9cd5be881ff0eea69402e7642d59d03075380 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Thu, 23 May 2024 17:45:01 +0200 Subject: [PATCH 08/80] Minor tsan fixes for the CPU smoke test --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 5 +++++ ddprof-lib/src/main/cpp/flightRecorder.cpp | 4 +--- ddprof-lib/src/main/cpp/spinLock.h | 2 +- ddprof-lib/src/main/cpp/thread.cpp | 4 ++++ ddprof-lib/src/main/cpp/threadFilter.cpp | 5 +++-- ddprof-lib/src/main/cpp/vmStructs.cpp | 4 ++++ 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 5b9c3b0b7..8f4ebe719 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -189,6 +189,9 @@ void CTimer::stop() { } void CTimer::signalHandler(int signo, siginfo_t* siginfo, void* ucontext) { + // Save the current errno value + int saved_errno = errno; + if (!_enabled) return; int tid = 0; ProfiledThread* current = ProfiledThread::current(); @@ -209,6 +212,8 @@ void CTimer::signalHandler(int signo, siginfo_t* siginfo, void* ucontext) { } Profiler::instance()->recordSample(ucontext, _interval, tid, BCI_CPU, 0, &event); Shims::instance().setSighandlerTid(-1); + // we need to avoid spoiling the value of errno (tsan report) + errno = saved_errno; } #endif // __linux__ diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index fa5c0bd36..2c9fbba29 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -1322,9 +1322,7 @@ void Recording::recordCpuLoad(Buffer* buf, float proc_user, float proc_system, f } void Recording::addThread(int tid) { - if (!_thread_set.accept(tid)) { - _thread_set.add(tid); - } + _thread_set.add(tid); } Error FlightRecorder::start(Arguments& args, bool reset) { diff --git a/ddprof-lib/src/main/cpp/spinLock.h b/ddprof-lib/src/main/cpp/spinLock.h index b472ed8f7..e5d7a81ac 100644 --- a/ddprof-lib/src/main/cpp/spinLock.h +++ b/ddprof-lib/src/main/cpp/spinLock.h @@ -51,7 +51,7 @@ class SpinLock { __sync_fetch_and_sub(&_lock, 1); } - bool SpinLock::tryLockShared() { + bool tryLockShared() { int value; while ((value = __atomic_load_n(&_lock, __ATOMIC_ACQUIRE)) <= 0) { if (__sync_bool_compare_and_swap(&_lock, value, value - 1)) { diff --git a/ddprof-lib/src/main/cpp/thread.cpp b/ddprof-lib/src/main/cpp/thread.cpp index dd59ce20a..540e35f74 100644 --- a/ddprof-lib/src/main/cpp/thread.cpp +++ b/ddprof-lib/src/main/cpp/thread.cpp @@ -42,6 +42,10 @@ void ProfiledThread::initExistingThreads() { pthread_once(&initialized, doInitExistingThreads); } +// The lifetime of this vector requires stronger guarantees. +// We need to ensure that the vector is not removed at the end of the process while threads are accessing it. +// This is to silence the sanitizer but should not be considered as a fix +__attribute__((no_sanitize("thread"))) void ProfiledThread::initCurrentThreadWithBuffer() { initTLSKey(); diff --git a/ddprof-lib/src/main/cpp/threadFilter.cpp b/ddprof-lib/src/main/cpp/threadFilter.cpp index 90acddf51..7e64ce02b 100644 --- a/ddprof-lib/src/main/cpp/threadFilter.cpp +++ b/ddprof-lib/src/main/cpp/threadFilter.cpp @@ -24,7 +24,7 @@ void trackPage() { Counters::increment(THREAD_FILTER_PAGES, 1); Counters::increment(THREAD_FILTER_BYTES, BITMAP_SIZE); } - + ThreadFilter::ThreadFilter() { _max_thread_id = OS::getMaxThreadId(128 * 1024); _max_bitmaps = (_max_thread_id + BITMAP_SIZE - 1) / BITMAP_SIZE; @@ -124,7 +124,8 @@ void ThreadFilter::collect(std::vector& v) { if (b != NULL) { int start_id = i * BITMAP_CAPACITY; for (int j = 0; j < BITMAP_SIZE / sizeof(u64); j++) { - u64 word = b[j]; + // Considering the functional impact, relaxed could be a reasonable order here + u64 word = __atomic_load_n(&b[j], __ATOMIC_ACQUIRE); while (word != 0) { v.push_back(start_id + j * 64 + __builtin_ctzl(word)); word &= (word - 1); diff --git a/ddprof-lib/src/main/cpp/vmStructs.cpp b/ddprof-lib/src/main/cpp/vmStructs.cpp index d2f2889c8..6d917dc3a 100644 --- a/ddprof-lib/src/main/cpp/vmStructs.cpp +++ b/ddprof-lib/src/main/cpp/vmStructs.cpp @@ -700,6 +700,10 @@ bool VMMethod::check_jmethodID_J9(jmethodID id) { return id != NULL && *((void**)id) != NULL; } +// We know that there is a race here. +// CodeHeap::allocate(unsigned long) can change the structure while we are accessing +// We can be defensive on the data we get from the map +__attribute__((no_sanitize("thread"))) NMethod* CodeHeap::findNMethod(char* heap, const void* pc) { unsigned char* heap_start = *(unsigned char**)(heap + _code_heap_memory_offset + _vs_low_offset); unsigned char* segmap = *(unsigned char**)(heap + _code_heap_segmap_offset + _vs_low_offset); From 826f86017ded36fcbcb3b6654c2dd169a7164589 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 14 May 2024 18:25:19 +0200 Subject: [PATCH 09/80] Allow new build/test configurations --- ddprof-lib/build.gradle | 189 +++++++++++++++++++++++++-------------- ddprof-test/build.gradle | 115 +++++++++++++++++++----- 2 files changed, 214 insertions(+), 90 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 1487c346c..9b76f598c 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -19,11 +19,15 @@ if (rootDir.toString().endsWith("ddprof-lib")) { apply from: rootProject.file('../common.gradle') } -def libraryTargetPath() { - return "${projectDir}/build/classes/java/main/META-INF/native-libs/${osIdentifier()}-${archIdentifier()}${isMusl() ? '-musl' : ''}" +def libraryTargetBase(type) { + return "${projectDir}/build/native/${type}" } -def librarySourcePath(boolean debug = false) { +def libraryTargetPath(type) { + return "${libraryTargetBase(type)}/META-INF/native-libs/${osIdentifier()}-${archIdentifier()}${isMusl() ? '-musl' : ''}" +} + +def librarySourcePath(type, qualifier = "") { def osarchext = "" if (osIdentifier() == 'linux' && archIdentifier() != 'x64') { // when built on aarch64 the location the library is built in is 'x86-64' ¯\_(ツ)_/¯ @@ -31,12 +35,11 @@ def librarySourcePath(boolean debug = false) { } else if (osIdentifier() == 'macos') { osarchext = archIdentifier() == 'x64' ? 'x86-64' : 'aarch64' } - def libsDir = debug ? "debug" : "release" - def qualifier = debug ? "" : "stripped" - return "${projectDir}/build/lib/main/${libsDir}/${osIdentifier()}/${osarchext}/${qualifier}/libjavaProfiler.${osIdentifier() == 'macos' ? 'dylib' : 'so'}" + return "${projectDir}/build/lib/main/${type}/${osIdentifier()}/${osarchext}/${qualifier}/libjavaProfiler.${osIdentifier() == 'macos' ? 'dylib' : 'so'}" } ext { + libraryTargetBase = this.&libraryTargetBase libraryTargetPath = this.&libraryTargetPath librarySourcePath = this.&librarySourcePath } @@ -46,14 +49,54 @@ targetCompatibility = JavaVersion.VERSION_1_8 def isGitlabCI = System.getenv("GITLAB_CI") != null +// if a new configuration is added and it is meant to be published to maven registry don't forget to update the 'artifacts' block +configurations { + assembled { + canBeConsumed = true + canBeResolved = false + extendsFrom implementation + } + release { + canBeConsumed = true + canBeResolved = false + extendsFrom implementation + } + debug { + canBeConsumed = true + canBeResolved = false + extendsFrom implementation + } + sanitized { + canBeConsumed = true + canBeResolved = false + extendsFrom implementation + } +} + +// Configure sanitized build type +tasks.register('compileSanitized', CppCompile) { + group = 'build' + description = 'Compile the sanitized build of the library' + objectFileDir = file("$buildDir/obj/main/sanitized") +} + +tasks.register('linkSanitized', LinkSharedLibrary) { + group = 'build' + description = 'Link the sanitized build of the library' + source = fileTree("$buildDir/obj/main/sanitized") + linkedFile = file("$buildDir/lib/main/sanitized/${osIdentifier()}/libjavaProfiler.so") + dependsOn compileSanitized +} + // configure the compiler here tasks.withType(CppCompile).configureEach { onlyIf { !project.hasProperty('skip-native') } + def taskIncludes = ["${javaHome()}/include"] def taskArgs = ["-fno-omit-frame-pointer", "-momit-leaf-frame-pointer", "-fvisibility=hidden", - "-fdata-sections", "-ffunction-sections", "-std=c++11", "-DPROFILER_VERSION=\"${version}\"", "-DCOUNTERS"] + "-fdata-sections", "-ffunction-sections", "-std=c++11", "-DPROFILER_VERSION=\"${version}\"", "-DCOUNTERS"] if (os().isMacOsX()) { taskIncludes.add "${javaHome()}/include/darwin" @@ -66,37 +109,33 @@ tasks.withType(CppCompile).configureEach { } if (it.name.contains('Debug')) { - taskArgs.add("-DDEBUG") - -// /usr/bin/c++ -fsanitize=undefined -fsanitize=float-divide-by-zero -fno-sanitize-recover -fsanitize=address -fstack-protector-all -std=gnu++20 -MD -MT CMakeFiles/test.dir/main.o -MF CMakeFiles/test.dir/main.o.d -o CMakeFiles/test.dir/main.o -c /home/r1viollet/dd/system-experiments/segvs/main.cpp - - if (os().isLinux()) { - taskArgs.add "-g" - taskArgs.add "-fsanitize=address" - taskArgs.add "-fsanitize=undefined" - // static could be interesting - // taskArgs.add "-static-libasan" - // This one we should test - // -fsanitize-address-use-after-return=always - taskArgs.add "-fno-sanitize-recover=all" - taskArgs.add "-fsanitize=float-divide-by-zero" - taskArgs.add "-fstack-protector-all" - + taskArgs.addAll(["-g", "-DDEBUG"]) } - } else { - taskArgs.add("-DNDEBUG") + if (it.name.contains("Sanitized")) { + if (os().isLinux()) { + // TODO: This one we should test: -fsanitize-address-use-after-return=always + taskArgs.addAll(['-g', '-DDEBUG', '-fPIC', '-fsanitize=address', '-fsanitize=undefined', '-fno-sanitize-recover=all', '-fsanitize=float-divide-by-zero', '-fstack-protector-all']) + } } - println "Compiler args: ${taskArgs}" - // TODO for Clang we would use scan-build utility instead but it needs to be installed first - + taskIncludes.add "${projectDir}/../malloc-shim/src/main/public" includes { taskIncludes } + if (it.name.contains("Release")) { + def sanitized = tasks.named('compileSanitized').get() + sanitized.toolChain = it.toolChain + sanitized.targetPlatform = it.targetPlatform + sanitized.includes = it.includes + sanitized.systemIncludes = it.systemIncludes + sanitized.source = it.source + taskArgs.add("-DNDEBUG") + } + compilerArgs.addAll(taskArgs) } @@ -108,28 +147,29 @@ tasks.withType(LinkSharedLibrary) { } def taskArgs = [] if (os().isLinux()) { - taskArgs.addAll(["-ldl", "-Wl,-z,defs", "--verbose", "-lpthread", "-lm", "-v"]) // Common flags - if (it.name.contains('Debug')) { - println "Adding debug-specific linker args" + taskArgs.addAll(["-ldl", "-Wl,-z,defs", "--verbose", "-lpthread", "-lm", "-lrt", "-v"]) // Common flags + if (it.name.contains('Sanitized')) { taskArgs.addAll([ - "-fsanitize=address", - "-fsanitize=undefined", - // "-static-libasan", // Ensure the ASan runtime is linked statically - "-fno-omit-frame-pointer" // Often recommended with sanitizers + "-fsanitize=address", + "-fsanitize=undefined", + // "-static-libasan", // Ensure the ASan runtime is linked statically + "-fno-omit-frame-pointer" // Often recommended with sanitizers ]) - } else { - println "Adding release-specific linker args" + } + if (it.name.contains("Release")) { taskArgs.addAll([ - "-Wl,-z,nodelete", - "-lrt", - "-static-libstdc++", - "-static-libgcc", - "-Wl,--exclude-libs,ALL", - "-Wl,--gc-sections" + "-Wl,-z,nodelete", + "-static-libstdc++", + "-static-libgcc", + "-Wl,--exclude-libs,ALL", + "-Wl,--gc-sections" ]) + def sanitized = tasks.named('linkSanitized').get() + sanitized.toolChain = it.toolChain + sanitized.targetPlatform = it.targetPlatform + sanitized.libs = it.libs } } - println "Linker args: ${taskArgs}" linkerArgs.addAll(taskArgs) } @@ -150,7 +190,6 @@ library { linkage = [Linkage.SHARED] } - dependencies { if (os().isLinux()) { // the malloc shim works only on linux @@ -173,8 +212,8 @@ task cppTest(type: Exec) { task copyLibs(type: Copy) { dependsOn cppTest if (!project.hasProperty("with-libs")) { - from file(librarySourcePath()) - into file(libraryTargetPath()) + from file(librarySourcePath("release", "stripped")) + into file(libraryTargetPath("release")) } } @@ -183,8 +222,19 @@ task copyDebugLibs(type: Copy) { // and keeps on complaining about undeclared inputs dependsOn copyLibs - from file(librarySourcePath(true)) - into file(libraryTargetPath()) + from file(librarySourcePath("debug")) + into file(libraryTargetPath("debug")) +} + +task copySanitizedLibs(type: Copy) { + // This must depend on copyLibs as Gradle detects that this task is using the same target path + // and keeps on complaining about undeclared inputs + dependsOn copyLibs + + dependsOn linkSanitized + + from file(librarySourcePath("sanitized")) + into file(libraryTargetPath("sanitized")) } // Allow specifying the external location for the native libraries @@ -227,7 +277,7 @@ task assembleJar(type: Jar) { archiveBaseName = libraryName archiveClassifier = "" archiveVersion = component_version - from sourceSets.main.output.classesDirs + from sourceSets.main.output.classesDirs + file(libraryTargetBase("release")) } task assembleDebugJar(type: Jar) { @@ -237,16 +287,26 @@ task assembleDebugJar(type: Jar) { archiveBaseName = libraryName archiveClassifier = "debug" archiveVersion = component_version - from sourceSets.main.output.classesDirs + from sourceSets.main.output.classesDirs + file(libraryTargetBase("debug")) +} + +task assembleSanitizedJar(type: Jar) { + if (!project.hasProperty('skip-native')) { + dependsOn copySanitizedLibs + } + archiveBaseName = libraryName + archiveClassifier = "sanitized" + archiveVersion = component_version + from sourceSets.main.output.classesDirs + file(libraryTargetBase("sanitized")) } task scanBuild(type: Exec) { workingDir "${projectDir}/src/test/make" commandLine 'scan-build' args "-o", "${projectDir}/build/reports/scan-build", - "--force-analyze-debug-code", - "--use-analyzer", "/usr/bin/clang++", - "make", "-j4", "clean", "all" + "--force-analyze-debug-code", + "--use-analyzer", "/usr/bin/clang++", + "make", "-j4", "clean", "all" } test { @@ -267,26 +327,15 @@ tasks.withType(Test) { executable = new File("${javaHome}", 'bin/java') } -configurations { - assembled { - canBeConsumed = true - canBeResolved = false - extendsFrom implementation - } - debug { - canBeConsumed = true - canBeResolved = false - extendsFrom implementation - } -} - artifacts { assembled(assembleJar) assembled(assembleDebugJar) + assembled(assembleSanitizedJar) assembled(sourcesJar) assembled(javadocJar) - assembled(assembleDebugJar) + release(assembleJar) debug(assembleDebugJar) + sanitized(assembleSanitizedJar) } publishing { @@ -294,6 +343,7 @@ publishing { assembled(MavenPublication) { publication -> publication.artifact project.tasks.assembleJar publication.artifact project.tasks.assembleDebugJar + publication.artifact project.tasks.assembleSanitizedJar publication.artifact sourcesJar publication.artifact javadocJar @@ -361,6 +411,9 @@ gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph -> afterEvaluate { assert description: "Project $project.path is published, must have a description" + // apparently, the compile/link tasks are a bit special and the dependency can be only added at the very end + tasks.named('compileSanitized').get().dependsOn tasks.named('compileReleaseLinuxCpp').get() + tasks.named('linkSanitized').get().dependsOn tasks.named('linkReleaseLinux').get() } // we are publishing very customized artifacts - we are attaching the native library to the resulting JAR artifact diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index 3f7fa11d4..ffd5f5963 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -6,47 +6,118 @@ repositories { mavenCentral() } -dependencies { - testImplementation project(path: ":ddprof-lib", configuration: 'debug') - testImplementation project(path: ":ddprof-test-tracer") +def locateLibasan() { + if (os().isLinux()) { + def locateCommand = 'locate libasan.so' + def process = locateCommand.execute() + process.waitFor() + + if (process.exitValue() == 0) { + def output = process.in.text + def lines = output.readLines() + def exactMatch = lines.find { it ==~ /.*libasan\.so$/ } + + if (exactMatch) { + return exactMatch + } + } + } + return null +} + +ext { + locateLibasan = this.&locateLibasan +} + +configurations { + testCommon { + canBeConsumed = true + canBeResolved = true + } + testDebugImplementation { + canBeConsumed = true + canBeResolved = true + extendsFrom testCommon + } + testSanitizedImplementation{ + canBeConsumed = true + canBeResolved = true + extendsFrom testCommon + } +} + +def addCommonTestDependencies(Configuration configuration) { + configuration.dependencies.add(project.dependencies.create('org.junit.jupiter:junit-jupiter-api:5.9.2')) + configuration.dependencies.add(project.dependencies.create('org.junit.jupiter:junit-jupiter-engine:5.9.2')) + configuration.dependencies.add(project.dependencies.create('org.junit.jupiter:junit-jupiter-params:5.9.2')) + configuration.dependencies.add(project.dependencies.create('org.slf4j:slf4j-simple:1.7.32')) + configuration.dependencies.add(project.dependencies.create('org.openjdk.jmc:flightrecorder:8.1.0')) + configuration.dependencies.add(project.dependencies.create('org.openjdk.jol:jol-core:0.16')) + configuration.dependencies.add(project.dependencies.create('org.junit-pioneer:junit-pioneer:1.9.1')) + configuration.dependencies.add(project.dependencies.create('org.lz4:lz4-java:1.8.0')) + configuration.dependencies.add(project.dependencies.create('org.xerial.snappy:snappy-java:1.1.10.1')) + configuration.dependencies.add(project.dependencies.create('com.github.luben:zstd-jni:1.5.5-4')) + configuration.dependencies.add(project.dependencies.project(path: ":ddprof-test-tracer")) +} + +addCommonTestDependencies(configurations.testImplementation) +addCommonTestDependencies(configurations.testDebugImplementation) +addCommonTestDependencies(configurations.testSanitizedImplementation) - // https://mvnrepository.com/artifact/org.junit.jupiter/junit-jupiter-api - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.2' - testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.9.2' - testImplementation 'org.junit.jupiter:junit-jupiter-params:5.9.2' - testImplementation 'org.slf4j:slf4j-simple:1.7.32' - testImplementation 'org.openjdk.jmc:flightrecorder:8.1.0' - testImplementation 'org.openjdk.jol:jol-core:0.16' - // https://mvnrepository.com/artifact/org.junit-pioneer/junit-pioneer - testImplementation group: 'org.junit-pioneer', name: 'junit-pioneer', version: '1.9.1' +dependencies { + testSanitizedImplementation project(path: ":ddprof-lib", configuration: 'sanitized') + testDebugImplementation project(path: ":ddprof-lib", configuration: 'debug') - testImplementation group: 'org.lz4', name: 'lz4-java', version: '1.8.0' - testImplementation group: 'org.xerial.snappy', name: 'snappy-java', version: '1.1.10.1' - testImplementation group: 'com.github.luben', name: 'zstd-jni', version: '1.5.5-4' + testImplementation project(path: ":ddprof-lib", configuration: 'release') +} +task testDebug(type: Test) { + dependsOn compileTestJava + description = 'Runs the unit tests with the debug library' + group = 'verification' + // Filter classpath to include only necessary dependencies for testSanitized + classpath = sourceSets.test.runtimeClasspath.filter { file -> + // Replace the condition with your actual filtering logic + !file.name.contains('ddprof-') || file.name.contains('test-tracer') + } + configurations.testDebugImplementation } -test { +task testSanitized(type: Test) { + dependsOn compileTestJava + def libasan = locateLibasan() onlyIf { - !project.hasProperty('skip-tests') + libasan != null } - useJUnitPlatform() - jvmArgs '-Djdk.attach.allowAttachSelf', '-Djol.tryWithSudo=true' + environment "LD_PRELOAD", libasan + environment "ASAN_OPTIONS", "detect_leaks=0,suppressions=${projectDir}/asan.supp" + environment "UBSAN_OPTIONS", "detect_leaks=0,suppressions=${projectDir}/ubsan.supp" + + description = 'Runs the unit tests with the sanitized library' + group = 'verification' + // Filter classpath to include only necessary dependencies for testSanitized + classpath = sourceSets.test.runtimeClasspath.filter { file -> + // Replace the condition with your actual filtering logic + !file.name.contains('ddprof-') || file.name.contains('test-tracer') + } + configurations.testSanitizedImplementation } +test.dependsOn testDebug + tasks.withType(Test) { - environment "LD_PRELOAD", "/usr/lib/gcc/x86_64-linux-gnu/11/libasan.so" - environment "ASAN_OPTIONS", "detect_leaks=0,suppressions=${projectDir}/asan.supp" - environment "UBSAN_OPTIONS", "detect_leaks=0,suppressions=${projectDir}/ubsan.supp" + // this is a shared configuration for all test tasks onlyIf { !project.hasProperty('skip-tests') } + + jvmArgs '-Djdk.attach.allowAttachSelf', '-Djol.tryWithSudo=true' + def javaHome = System.getenv("JAVA_TEST_HOME") if (javaHome == null) { javaHome = System.getenv("JAVA_HOME") } + useJUnitPlatform() executable = new File("${javaHome}", 'bin/java') } From f6906b929c3f7290195be249364c60f9500db004 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 14 May 2024 18:25:19 +0200 Subject: [PATCH 10/80] Add build comments --- ddprof-lib/build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 9b76f598c..fc822dfd2 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -127,6 +127,8 @@ tasks.withType(CppCompile).configureEach { } if (it.name.contains("Release")) { + // need to configure the compile tasks for other configurations here - + // the 'compileReleaseLinuxCpp' task is configured on-the-fly and as such we need to wait till it is fully configured def sanitized = tasks.named('compileSanitized').get() sanitized.toolChain = it.toolChain sanitized.targetPlatform = it.targetPlatform @@ -164,6 +166,8 @@ tasks.withType(LinkSharedLibrary) { "-Wl,--exclude-libs,ALL", "-Wl,--gc-sections" ]) + // need to configure the link tasks for other configurations here - + // the 'linkReleaseLinux' task is configured on-the-fly and as such we need to wait till it is fully configured def sanitized = tasks.named('linkSanitized').get() sanitized.toolChain = it.toolChain sanitized.targetPlatform = it.targetPlatform From b63c5de497346e3f0169dc1c0f855ce474f3b286 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Fri, 24 May 2024 21:37:05 +0200 Subject: [PATCH 11/80] Minor adjustements in sanitizer flags --- ddprof-lib/build.gradle | 21 ++++++++++++++++++--- ddprof-lib/src/main/cpp/buffers.h | 3 +++ ddprof-lib/src/main/cpp/dwarf.h | 2 +- ddprof-test/build.gradle | 10 +++++++--- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index fc822dfd2..95b88ce6d 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -113,8 +113,24 @@ tasks.withType(CppCompile).configureEach { } if (it.name.contains("Sanitized")) { if (os().isLinux()) { - // TODO: This one we should test: -fsanitize-address-use-after-return=always - taskArgs.addAll(['-g', '-DDEBUG', '-fPIC', '-fsanitize=address', '-fsanitize=undefined', '-fno-sanitize-recover=all', '-fsanitize=float-divide-by-zero', '-fstack-protector-all']) + taskArgs.addAll([ + '-g', // Generate debug information + '-DDEBUG', // Enable debug mode + '-fPIC', // Position-independent code + '-fsanitize=address', // AddressSanitizer for memory errors + '-fsanitize=undefined', // UndefinedBehaviorSanitizer for undefined behavior + '-fno-sanitize-recover=all', // Disable recovery to ensure program halts on error + '-fsanitize=float-divide-by-zero', // Catch floating point division by zero + '-fstack-protector-all', // Protect against stack buffer overflows + '-fsanitize=leak', // LeakSanitizer to detect memory leaks + '-fsanitize=pointer-overflow', // Catch pointer overflows + '-fsanitize=return', // Check for uninitialized return values + '-fsanitize=bounds', // BoundsSanitizer for array bounds + '-fsanitize=alignment', // Catch misaligned pointer access + '-fsanitize=object-size', // Catch accesses that are likely out of bounds + '-fno-omit-frame-pointer', // Keep frame pointer for better debugging + '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces + ]) } } @@ -154,7 +170,6 @@ tasks.withType(LinkSharedLibrary) { taskArgs.addAll([ "-fsanitize=address", "-fsanitize=undefined", - // "-static-libasan", // Ensure the ASan runtime is linked statically "-fno-omit-frame-pointer" // Often recommended with sanitizers ]) } diff --git a/ddprof-lib/src/main/cpp/buffers.h b/ddprof-lib/src/main/cpp/buffers.h index e221fda10..f45ae5a55 100644 --- a/ddprof-lib/src/main/cpp/buffers.h +++ b/ddprof-lib/src/main/cpp/buffers.h @@ -87,6 +87,9 @@ class Buffer { _offset += 2; } + // java-profiler/ddprof-lib/src/main/cpp/buffers.h:92:34: runtime error: store to misaligned address 0x7f3c446ec81e for type 'int', which requires 4 byte alignment + // 0x7f3c446ec81e: note: pointer points here + __attribute__((no_sanitize("undefined"))) void put32(int v) { assert(_offset + 4 < limit()); *(int*)(_data + _offset) = htonl(v); diff --git a/ddprof-lib/src/main/cpp/dwarf.h b/ddprof-lib/src/main/cpp/dwarf.h index 72788d7c2..ab000a985 100644 --- a/ddprof-lib/src/main/cpp/dwarf.h +++ b/ddprof-lib/src/main/cpp/dwarf.h @@ -119,7 +119,7 @@ class DwarfParser { return *(u16*)add(2); } - // same issue + // same issue as in get16 __attribute__((no_sanitize("undefined"))) u32 get32() { return *(u32*)add(4); diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index ffd5f5963..94e6b0ebd 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -8,7 +8,8 @@ repositories { def locateLibasan() { if (os().isLinux()) { - def locateCommand = 'locate libasan.so' + // Locate libasan.so + def locateCommand = 'gcc -print-file-name=libasan.so' def process = locateCommand.execute() process.waitFor() @@ -91,8 +92,11 @@ task testSanitized(type: Test) { } environment "LD_PRELOAD", libasan - environment "ASAN_OPTIONS", "detect_leaks=0,suppressions=${projectDir}/asan.supp" - environment "UBSAN_OPTIONS", "detect_leaks=0,suppressions=${projectDir}/ubsan.supp" + // warning: stack use after return can cause slowness on arm64 + environment "ASAN_OPTIONS", "detect_stack_use_after_return=1 suppressions=${projectDir}/asan.supp" + environment "UBSAN_OPTIONS", "halt_on_error=1 abort_on_error=1 print_stacktrace=1 suppressions=${projectDir}/ubsan.supp" + // lsan still does not run for all tests - manually trigger on some tests + environment "LSAN_OPTIONS", "detect_leaks=0" description = 'Runs the unit tests with the sanitized library' group = 'verification' From a44f570396a2ff41f3e9d889ff1eff1ae6b1ded3 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Sat, 25 May 2024 19:08:46 +0200 Subject: [PATCH 12/80] Rework the build configurations --- build.gradle | 2 + ddprof-lib/build.gradle | 393 +++++++++++++++----------------- ddprof-test-tracer/build.gradle | 2 +- ddprof-test/build.gradle | 136 +++++------ gradle/configurations.gradle | 134 +++++++++++ 5 files changed, 378 insertions(+), 289 deletions(-) create mode 100644 gradle/configurations.gradle diff --git a/build.gradle b/build.gradle index 38bf8ae5a..0ccde5df5 100644 --- a/build.gradle +++ b/build.gradle @@ -41,6 +41,7 @@ allprojects { group = 'com.datadoghq' apply from: rootProject.file('common.gradle') + apply from: rootProject.file('gradle/configurations.gradle') } subprojects { @@ -48,6 +49,7 @@ subprojects { } apply from: rootProject.file('common.gradle') +apply from: rootProject.file('gradle/configurations.gradle') def isCI = System.getenv("CI") != null diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 95b88ce6d..3fef0d5ba 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -7,6 +7,7 @@ plugins { id 'com.github.ben-manes.versions' version '0.27.0' id "com.diffplug.spotless" version "6.11.0" } + def libraryName = "ddprof" description = "Datadog Java Profiler Library" @@ -24,7 +25,14 @@ def libraryTargetBase(type) { } def libraryTargetPath(type) { - return "${libraryTargetBase(type)}/META-INF/native-libs/${osIdentifier()}-${archIdentifier()}${isMusl() ? '-musl' : ''}" + def osarchext = "" + if (osIdentifier() == 'linux' && archIdentifier() != 'x64') { + // when built on aarch64 the location the library is built in is 'x86-64' ¯\_(ツ)_/¯ + osarchext = "x86-64" + } else if (osIdentifier() == 'macos') { + osarchext = archIdentifier() == 'x64' ? 'x86-64' : 'arm64' + } + return "${libraryTargetBase(type)}/META-INF/native-libs/${osIdentifier()}-${osarchext}${isMusl() ? '-musl' : ''}" } def librarySourcePath(type, qualifier = "") { @@ -33,7 +41,7 @@ def librarySourcePath(type, qualifier = "") { // when built on aarch64 the location the library is built in is 'x86-64' ¯\_(ツ)_/¯ osarchext = "x86-64" } else if (osIdentifier() == 'macos') { - osarchext = archIdentifier() == 'x64' ? 'x86-64' : 'aarch64' + osarchext = archIdentifier() == 'x64' ? 'x86-64' : 'arm64' } return "${projectDir}/build/lib/main/${type}/${osIdentifier()}/${osarchext}/${qualifier}/libjavaProfiler.${osIdentifier() == 'macos' ? 'dylib' : 'so'}" } @@ -49,49 +57,174 @@ targetCompatibility = JavaVersion.VERSION_1_8 def isGitlabCI = System.getenv("GITLAB_CI") != null -// if a new configuration is added and it is meant to be published to maven registry don't forget to update the 'artifacts' block -configurations { - assembled { - canBeConsumed = true - canBeResolved = false - extendsFrom implementation +// Allow specifying the external location for the native libraries +// The libraries should be properly sorted into subfolders corresponding to the `libraryTargetPath` value for each +// os/arch/libc combination +tasks.register('copyExternalLibs', Copy) { + if (project.hasProperty("with-libs")) { + from(project.getProperty("with-libs")) { + include "**/*.so" + include "**/*.dylib" + } + into "${projectDir}/build/classes/java/main/META-INF/native-libs" } - release { - canBeConsumed = true - canBeResolved = false - extendsFrom implementation +} + +// the bridge between the gradle build and gtests +// the gradle gtest plugin(s) are quite unusable at this moment - using a shell script seems to be the only sane option +tasks.register('cppTest', Exec) { + if (project.hasProperty('skip-tests') || project.hasProperty('skip-cpp-tests')) { + environment 'SKIP_TESTS', 'true' } - debug { + workingDir "$projectDir" + commandLine './test.sh' + + dependsOn assemble +} + +tasks.register('assembleAll') {} + +// use the build config names to create configurations, copy lib and asemble jar tasks + +buildConfigNames().each { name -> + configurations.create(name) { canBeConsumed = true canBeResolved = false - extendsFrom implementation + extendsFrom configurations.implementation + } + def copyTask = tasks.register("copy${capitalizeFirstLetter(name)}Libs", Copy) { + dependsOn cppTest + from file(librarySourcePath(name, name == 'release' ? 'stripped' : '')).parent // the release build is stripped + into file(libraryTargetPath(name)) } - sanitized { + + def assembleTask = tasks.register("assemble${capitalizeFirstLetter(name)}Jar", Jar) { + group = 'build' + description = "Assemble the ${name} build of the library" + dependsOn copyExternalLibs + if (!project.hasProperty('skip-native')) { + dependsOn copyTask // tasks.named("copy${capitalizeFirstLetter(name)}Libs") + } + from sourceSets.main.output.classesDirs + from files(libraryTargetBase(name)) { + include "**/*" + } + archiveBaseName = libraryName + archiveClassifier = name == 'release' ? '' : name // the release qualifier is empty + archiveVersion = component_version + } + + tasks.assembleAll.dependsOn assembleTask +} +configurations { + // the 'all' configuration is used to aggregate all the build configurations + assembled { canBeConsumed = true canBeResolved = false extendsFrom implementation } } -// Configure sanitized build type -tasks.register('compileSanitized', CppCompile) { - group = 'build' - description = 'Compile the sanitized build of the library' - objectFileDir = file("$buildDir/obj/main/sanitized") +// we need this trickery to reuse the toolchain and system config from tasks created by the cpp-library plugin +tasks.whenTaskAdded { task -> + if (task instanceof CppCompile) { + if (!task.name.startsWith('compileLib') && task.name.contains('Release')) { + buildConfigurations.each { config -> + if (config.os == osIdentifier() && config.arch == archIdentifier()) { + def thisTask = tasks.register("compileLib${capitalizeFirstLetter(config.name)}", CppCompile) { + group = 'build' + description = "Compile the ${config.name} build of the library" + objectFileDir = file("$buildDir/obj/main/${config.name}") + compilerArgs.addAll(config.compilerArgs) + toolChain = task.toolChain + targetPlatform = task.targetPlatform + includes = task.includes + systemIncludes = task.systemIncludes + source = task.source + } + def linkTask = tasks.findByName("linkLib${capitalizeFirstLetter(config.name)}".toString()) + if (linkTask != null) { + linkTask.dependsOn thisTask + } + } + } + } + } else if (task instanceof LinkSharedLibrary) { + if (!task.name.startsWith('linkLib') && task.name.contains('Release')) { + buildConfigurations.each { config -> + if (config.os == osIdentifier() && config.arch == archIdentifier()) { + def thisTask = tasks.register("linkLib${capitalizeFirstLetter(config.name)}", LinkSharedLibrary) { + group = 'build' + description = "Link the ${config.name} build of the library" + source = fileTree("$buildDir/obj/main/${config.name}") + linkedFile = file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/libjavaProfiler.${os().isLinux() ? 'so' : 'dylib'}") + def compileTask = tasks.findByName("compileLib${capitalizeFirstLetter(config.name)}".toString()) + if (compileTask != null) { + dependsOn compileTask + } + linkerArgs.addAll(config.linkerArgs) + toolChain = task.toolChain + targetPlatform = task.targetPlatform + libs = task.libs + } + if (config.name == 'release') { + tasks.register('stripLibRelease', StripSymbols) { + dependsOn thisTask + targetPlatform = tasks.linkLibRelease.targetPlatform + toolChain = tasks.linkLibRelease.toolChain + binaryFile = tasks.linkLibRelease.linkedFile.get() + outputFile = file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/stripped/libjavaProfiler.${os().isLinux() ? 'so' : 'dylib'}") + } + } + } + } + } + } } -tasks.register('linkSanitized', LinkSharedLibrary) { - group = 'build' - description = 'Link the sanitized build of the library' - source = fileTree("$buildDir/obj/main/sanitized") - linkedFile = file("$buildDir/lib/main/sanitized/${osIdentifier()}/libjavaProfiler.so") - dependsOn compileSanitized +gradle.projectsEvaluated { + tasks.copyReleaseLibs.dependsOn tasks.stripLibRelease + + buildConfigNames().each { + if (it != 'release') { + def linkTask = tasks.findByName("linkLib${capitalizeFirstLetter(it)}") + def copyTask = tasks.findByName("copy${capitalizeFirstLetter(it)}Libs") + if (linkTask != null) { + copyTask.dependsOn linkTask + } + } + } +// // We need to introduce dependencies between generated linker and copy tasks +// // Find all tasks named 'copy*Libs' +// def copyLibsTasks = tasks.matching { +// it.name.startsWith('copy') && it.name.endsWith('Libs') && !it.name.contains('Release') +// } +// // Find all tasks named 'linkLib*' +// def linkLibTasks = tasks.matching { it.name == 'linkLib') } +// +// // For each 'copy*Libs' task, set dependencies on 'linkLib*' tasks +// copyLibsTasks.all { copyTask -> +// linkLibTasks.matching { +// +// }.all { linkTask -> +// copyTask.dependsOn(linkTask) +// println "Configured ${copyTask.name} to depend on ${linkTask.name}" +// } +// } } // configure the compiler here tasks.withType(CppCompile).configureEach { - onlyIf { - !project.hasProperty('skip-native') + if (name.startsWith('compileLib')) { + onlyIf { + !project.hasProperty('skip-native') + } + } else { + onlyIf { + // enable the built-in compiler task only for release; required by the stripsymbols task +// name.contains("Release") + false + } } def taskIncludes = ["${javaHome()}/include"] @@ -108,32 +241,6 @@ tasks.withType(CppCompile).configureEach { } } - if (it.name.contains('Debug')) { - taskArgs.addAll(["-g", "-DDEBUG"]) - } - if (it.name.contains("Sanitized")) { - if (os().isLinux()) { - taskArgs.addAll([ - '-g', // Generate debug information - '-DDEBUG', // Enable debug mode - '-fPIC', // Position-independent code - '-fsanitize=address', // AddressSanitizer for memory errors - '-fsanitize=undefined', // UndefinedBehaviorSanitizer for undefined behavior - '-fno-sanitize-recover=all', // Disable recovery to ensure program halts on error - '-fsanitize=float-divide-by-zero', // Catch floating point division by zero - '-fstack-protector-all', // Protect against stack buffer overflows - '-fsanitize=leak', // LeakSanitizer to detect memory leaks - '-fsanitize=pointer-overflow', // Catch pointer overflows - '-fsanitize=return', // Check for uninitialized return values - '-fsanitize=bounds', // BoundsSanitizer for array bounds - '-fsanitize=alignment', // Catch misaligned pointer access - '-fsanitize=object-size', // Catch accesses that are likely out of bounds - '-fno-omit-frame-pointer', // Keep frame pointer for better debugging - '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces - ]) - } - } - // TODO for Clang we would use scan-build utility instead but it needs to be installed first taskIncludes.add "${projectDir}/../malloc-shim/src/main/public" @@ -142,60 +249,23 @@ tasks.withType(CppCompile).configureEach { taskIncludes } - if (it.name.contains("Release")) { - // need to configure the compile tasks for other configurations here - - // the 'compileReleaseLinuxCpp' task is configured on-the-fly and as such we need to wait till it is fully configured - def sanitized = tasks.named('compileSanitized').get() - sanitized.toolChain = it.toolChain - sanitized.targetPlatform = it.targetPlatform - sanitized.includes = it.includes - sanitized.systemIncludes = it.systemIncludes - sanitized.source = it.source - taskArgs.add("-DNDEBUG") - } - compilerArgs.addAll(taskArgs) } // configure linker -tasks.withType(LinkSharedLibrary) { - onlyIf { - !project.hasProperty('skip-native') - } - def taskArgs = [] - if (os().isLinux()) { - taskArgs.addAll(["-ldl", "-Wl,-z,defs", "--verbose", "-lpthread", "-lm", "-lrt", "-v"]) // Common flags - if (it.name.contains('Sanitized')) { - taskArgs.addAll([ - "-fsanitize=address", - "-fsanitize=undefined", - "-fno-omit-frame-pointer" // Often recommended with sanitizers - ]) +tasks.withType(LinkSharedLibrary).configureEach { + if (name.startsWith('linkLib')) { + onlyIf { + !project.hasProperty('skip-native') } - if (it.name.contains("Release")) { - taskArgs.addAll([ - "-Wl,-z,nodelete", - "-static-libstdc++", - "-static-libgcc", - "-Wl,--exclude-libs,ALL", - "-Wl,--gc-sections" - ]) - // need to configure the link tasks for other configurations here - - // the 'linkReleaseLinux' task is configured on-the-fly and as such we need to wait till it is fully configured - def sanitized = tasks.named('linkSanitized').get() - sanitized.toolChain = it.toolChain - sanitized.targetPlatform = it.targetPlatform - sanitized.libs = it.libs + } else { + onlyIf { + // enable the built-in linker task only for release; required by the stripsymbols task +// name.contains("Release") + false } } - linkerArgs.addAll(taskArgs) -} - -tasks.withType(StripSymbols) { - onlyIf { - !project.hasProperty('skip-native') - } } library { @@ -209,6 +279,12 @@ library { linkage = [Linkage.SHARED] } +tasks.withType(StripSymbols).configureEach { + onlyIf { + name == ("stripLibRelease") + } +} + dependencies { if (os().isLinux()) { // the malloc shim works only on linux @@ -216,69 +292,16 @@ dependencies { } } -// the bridge between the gradle build and gtests -// the gradle gtest plugin(s) are quite unusable at this moment - using a shell script seems to be the only sane option -task cppTest(type: Exec) { - if (project.hasProperty('skip-tests') || project.hasProperty('skip-cpp-tests')) { - environment 'SKIP_TESTS', 'true' - } - workingDir "$projectDir" - commandLine './test.sh' - - dependsOn assemble -} - -task copyLibs(type: Copy) { - dependsOn cppTest - if (!project.hasProperty("with-libs")) { - from file(librarySourcePath("release", "stripped")) - into file(libraryTargetPath("release")) - } -} - -task copyDebugLibs(type: Copy) { - // This must depend on copyLibs as Gradle detects that this task is using the same target path - // and keeps on complaining about undeclared inputs - dependsOn copyLibs - - from file(librarySourcePath("debug")) - into file(libraryTargetPath("debug")) -} - -task copySanitizedLibs(type: Copy) { - // This must depend on copyLibs as Gradle detects that this task is using the same target path - // and keeps on complaining about undeclared inputs - dependsOn copyLibs - - dependsOn linkSanitized - - from file(librarySourcePath("sanitized")) - into file(libraryTargetPath("sanitized")) -} - -// Allow specifying the external location for the native libraries -// The libraries should be properly sorted into subfolders corresponding to the `libraryTargetPath` value for each -// os/arch/libc combination -task copyExternalLibs(type: Copy) { - if (project.hasProperty("with-libs")) { - from(project.getProperty("with-libs")) { - include "**/*.so" - include "**/*.dylib" - } - into "${projectDir}/build/classes/java/main/META-INF/native-libs" - } -} - jar.dependsOn copyExternalLibs -task sourcesJar(type: Jar) { +tasks.register('sourcesJar', Jar) { from sourceSets.main.allJava archiveBaseName = libraryName archiveClassifier = "sources" archiveVersion = component_version } -task javadocJar(type: Jar) { +tasks.register('javadocJar', Jar) { dependsOn javadoc archiveBaseName = libraryName archiveClassifier = 'javadoc' @@ -286,40 +309,9 @@ task javadocJar(type: Jar) { from javadoc.destinationDir } -javadoc.dependsOn copyLibs +javadoc.dependsOn copyReleaseLibs -task assembleJar(type: Jar) { - dependsOn copyExternalLibs - if (!project.hasProperty('skip-native')) { - dependsOn copyLibs - } - archiveBaseName = libraryName - archiveClassifier = "" - archiveVersion = component_version - from sourceSets.main.output.classesDirs + file(libraryTargetBase("release")) -} - -task assembleDebugJar(type: Jar) { - if (!project.hasProperty('skip-native')) { - dependsOn copyDebugLibs - } - archiveBaseName = libraryName - archiveClassifier = "debug" - archiveVersion = component_version - from sourceSets.main.output.classesDirs + file(libraryTargetBase("debug")) -} - -task assembleSanitizedJar(type: Jar) { - if (!project.hasProperty('skip-native')) { - dependsOn copySanitizedLibs - } - archiveBaseName = libraryName - archiveClassifier = "sanitized" - archiveVersion = component_version - from sourceSets.main.output.classesDirs + file(libraryTargetBase("sanitized")) -} - -task scanBuild(type: Exec) { +tasks.register('scanBuild', Exec) { workingDir "${projectDir}/src/test/make" commandLine 'scan-build' args "-o", "${projectDir}/build/reports/scan-build", @@ -347,22 +339,20 @@ tasks.withType(Test) { } artifacts { - assembled(assembleJar) - assembled(assembleDebugJar) - assembled(assembleSanitizedJar) - assembled(sourcesJar) - assembled(javadocJar) - release(assembleJar) - debug(assembleDebugJar) - sanitized(assembleSanitizedJar) + // create artifacts for all configures build config names + buildConfigNames().each { + def task = tasks.named("assemble${capitalizeFirstLetter(it)}Jar") + artifacts.add('assembled', task) + artifacts.add(it, task) + } } publishing { publications { assembled(MavenPublication) { publication -> - publication.artifact project.tasks.assembleJar - publication.artifact project.tasks.assembleDebugJar - publication.artifact project.tasks.assembleSanitizedJar + buildConfigNames().each { + publication.artifact tasks.named("assemble${capitalizeFirstLetter(it)}Jar") + } publication.artifact sourcesJar publication.artifact javadocJar @@ -430,15 +420,12 @@ gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph -> afterEvaluate { assert description: "Project $project.path is published, must have a description" - // apparently, the compile/link tasks are a bit special and the dependency can be only added at the very end - tasks.named('compileSanitized').get().dependsOn tasks.named('compileReleaseLinuxCpp').get() - tasks.named('linkSanitized').get().dependsOn tasks.named('linkReleaseLinux').get() } // we are publishing very customized artifacts - we are attaching the native library to the resulting JAR artifact tasks.withType(AbstractPublishToMaven).configureEach { if (it.name.contains('AssembledPublication')) { - it.dependsOn assembleJar + it.dependsOn assembleReleaseJar } rootProject.subprojects { mustRunAfter tasks.matching { it instanceof VerificationTask } diff --git a/ddprof-test-tracer/build.gradle b/ddprof-test-tracer/build.gradle index 9aa7b6484..f9be95a3f 100644 --- a/ddprof-test-tracer/build.gradle +++ b/ddprof-test-tracer/build.gradle @@ -7,7 +7,7 @@ repositories { } dependencies { - implementation project(path: ":ddprof-lib", configuration: 'debug') + implementation project(path: ":ddprof-lib", configuration: 'release') } tasks.withType(JavaCompile).configureEach { diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index 94e6b0ebd..186750c12 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -6,47 +6,6 @@ repositories { mavenCentral() } -def locateLibasan() { - if (os().isLinux()) { - // Locate libasan.so - def locateCommand = 'gcc -print-file-name=libasan.so' - def process = locateCommand.execute() - process.waitFor() - - if (process.exitValue() == 0) { - def output = process.in.text - def lines = output.readLines() - def exactMatch = lines.find { it ==~ /.*libasan\.so$/ } - - if (exactMatch) { - return exactMatch - } - } - } - return null -} - -ext { - locateLibasan = this.&locateLibasan -} - -configurations { - testCommon { - canBeConsumed = true - canBeResolved = true - } - testDebugImplementation { - canBeConsumed = true - canBeResolved = true - extendsFrom testCommon - } - testSanitizedImplementation{ - canBeConsumed = true - canBeResolved = true - extendsFrom testCommon - } -} - def addCommonTestDependencies(Configuration configuration) { configuration.dependencies.add(project.dependencies.create('org.junit.jupiter:junit-jupiter-api:5.9.2')) configuration.dependencies.add(project.dependencies.create('org.junit.jupiter:junit-jupiter-engine:5.9.2')) @@ -61,55 +20,49 @@ def addCommonTestDependencies(Configuration configuration) { configuration.dependencies.add(project.dependencies.project(path: ":ddprof-test-tracer")) } -addCommonTestDependencies(configurations.testImplementation) -addCommonTestDependencies(configurations.testDebugImplementation) -addCommonTestDependencies(configurations.testSanitizedImplementation) - -dependencies { - testSanitizedImplementation project(path: ":ddprof-lib", configuration: 'sanitized') - testDebugImplementation project(path: ":ddprof-lib", configuration: 'debug') - - testImplementation project(path: ":ddprof-lib", configuration: 'release') +configurations.create('testCommon') { + canBeConsumed = true + canBeResolved = true } -task testDebug(type: Test) { - dependsOn compileTestJava - description = 'Runs the unit tests with the debug library' - group = 'verification' - // Filter classpath to include only necessary dependencies for testSanitized - classpath = sourceSets.test.runtimeClasspath.filter { file -> - // Replace the condition with your actual filtering logic - !file.name.contains('ddprof-') || file.name.contains('test-tracer') - } + configurations.testDebugImplementation - -} - -task testSanitized(type: Test) { - dependsOn compileTestJava - def libasan = locateLibasan() - onlyIf { - libasan != null +buildConfigurations.each { config -> + def name = config.name + if (config.os != osIdentifier() || config.arch != archIdentifier()) { + return + } + def cfg = configurations.create("test${capitalizeFirstLetter(name)}Implementation") { + canBeConsumed = true + canBeResolved = true + extendsFrom configurations.testCommon + } + addCommonTestDependencies(cfg) + cfg.dependencies.add(project.dependencies.project(path: ":ddprof-lib", configuration: name)) + + def task = tasks.register("test${name}", Test) { + dependsOn compileTestJava + description = "Runs the unit tests with the ${name} library variant" + group = 'verification' + // Filter classpath to include only necessary dependencies for this test task + classpath = sourceSets.test.runtimeClasspath.filter { file -> + !file.name.contains('ddprof-') || file.name.contains('test-tracer') + } + cfg + + + if (!config.testEnv.empty) { + println "applying env for test task test${capitalizeFirstLetter(name)}" + config.testEnv.each { key, value -> + println "setting env $key=$value for test task test${capitalizeFirstLetter(name)}" + environment key, value + } + } + } + if (name == 'debug') { + // redirect the default test task to the debug test task + tasks.test.dependsOn task.get() } - - environment "LD_PRELOAD", libasan - // warning: stack use after return can cause slowness on arm64 - environment "ASAN_OPTIONS", "detect_stack_use_after_return=1 suppressions=${projectDir}/asan.supp" - environment "UBSAN_OPTIONS", "halt_on_error=1 abort_on_error=1 print_stacktrace=1 suppressions=${projectDir}/ubsan.supp" - // lsan still does not run for all tests - manually trigger on some tests - environment "LSAN_OPTIONS", "detect_leaks=0" - - description = 'Runs the unit tests with the sanitized library' - group = 'verification' - // Filter classpath to include only necessary dependencies for testSanitized - classpath = sourceSets.test.runtimeClasspath.filter { file -> - // Replace the condition with your actual filtering logic - !file.name.contains('ddprof-') || file.name.contains('test-tracer') - } + configurations.testSanitizedImplementation } -test.dependsOn testDebug - -tasks.withType(Test) { +tasks.withType(Test).configureEach { // this is a shared configuration for all test tasks onlyIf { !project.hasProperty('skip-tests') @@ -123,6 +76,18 @@ tasks.withType(Test) { } useJUnitPlatform() executable = new File("${javaHome}", 'bin/java') + + if (name.contains("Asan")) { + onlyIf { + hasAsan() + } + } +} + +test { + onlyIf { + false // disable the default test task + } } tasks.withType(JavaCompile).configureEach { @@ -132,4 +97,5 @@ tasks.withType(JavaCompile).configureEach { sourceCompatibility = '8' targetCompatibility = '8' } + classpath += configurations.testReleaseImplementation } \ No newline at end of file diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle new file mode 100644 index 000000000..5a195df45 --- /dev/null +++ b/gradle/configurations.gradle @@ -0,0 +1,134 @@ +apply from: rootProject.file('common.gradle') + +ext.buildConfigurations = [] + +def static capitalizeFirstLetter(str) { + if (str == null || str.isEmpty()) { + return str + } + return str.substring(0, 1).toUpperCase() + str.substring(1) +} + +def buildConfigNames() { + buildConfigurations.findAll { + it.os == osIdentifier() && it.arch == archIdentifier() + }.collect { it.name }.toSet() +} + +def locateLibasan() { + if (os().isLinux()) { + try { + def locateCommand = 'locate libasan.so' + def process = locateCommand.execute() + process.waitFor() + + if (process.exitValue() == 0) { + def output = process.in.text + def lines = output.readLines() + def exactMatch = lines.find { it ==~ /.*libasan\.so$/ } + + if (exactMatch) { + return exactMatch + } + } + } catch (Throwable t) { + // ignored + } + } + return null +} + +def libasan = locateLibasan() + +def hasAsan() { + return libasan != null +} + +ext.addBuildConfiguration = { String name, String os, String arch, List compilerArgs, List linkerArgs, Map testEnv = [:] -> + buildConfigurations << [name: name, os: os, arch: arch, compilerArgs: compilerArgs, linkerArgs: linkerArgs, testEnv: testEnv] +} + +ext { + hasAsan = this.&hasAsan + capitalizeFirstLetter = this.&capitalizeFirstLetter + buildConfigNames = this.&buildConfigNames +} + +// ======= Define build configurations below ======== + +def commonLinuxLinkerArgs = ["-ldl", "-Wl,-z,defs", "--verbose", "-lpthread", "-lm", "-lrt", "-v"] + +def asanEnv = libasan != null ? + ['LD_PRELOAD': libasan, + // warning: stack use after return can cause slowness on arm64 + "ASAN_OPTIONS" : "detect_stack_use_after_return=1 suppressions=${projectDir}/asan.supp", + "UBSAN_OPTIONS" : "halt_on_error=1 abort_on_error=1 print_stacktrace=1 suppressions=${projectDir}/ubsan.supp", + // lsan still does not run for all tests - manually trigger on some tests + "LSAN_OPTIONS" : "detect_leaks=0" + ] : [:] + +def asanCompilerArgs = [ + '-g', + // Generate debug information + '-DDEBUG', + // Enable debug mode + '-fPIC', + // Position-independent code + '-fsanitize=address', + // AddressSanitizer for memory errors + '-fsanitize=undefined', + // UndefinedBehaviorSanitizer for undefined behavior + '-fno-sanitize-recover=all', + // Disable recovery to ensure program halts on error + '-fsanitize=float-divide-by-zero', + // Catch floating point division by zero + '-fstack-protector-all', + // Protect against stack buffer overflows + '-fsanitize=leak', + // LeakSanitizer to detect memory leaks + '-fsanitize=pointer-overflow', + // Catch pointer overflows + '-fsanitize=return', + // Check for uninitialized return values + '-fsanitize=bounds', + // BoundsSanitizer for array bounds + '-fsanitize=alignment', + // Catch misaligned pointer access + '-fsanitize=object-size', + // Catch accesses that are likely out of bounds + '-fno-omit-frame-pointer', + // Keep frame pointer for better debugging + '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces +] + +def asanLinkerArgs = [ + "-fsanitize=address", + "-fsanitize=undefined", + "-fno-omit-frame-pointer" // Often recommended with sanitizers +] + +// Linux +addBuildConfiguration 'release', 'linux', 'x64', + ['-O3', '-DNDEBUG'], + commonLinuxLinkerArgs + [ + "-Wl,-z,nodelete", + "-static-libstdc++", + "-static-libgcc", + "-Wl,--exclude-libs,ALL", + "-Wl,--gc-sections" + ] +addBuildConfiguration 'debug', 'linux', 'x64', + ['-O0', '-g', '-DDEBUG'], + commonLinuxLinkerArgs +addBuildConfiguration 'asan', 'linux', 'x64', + asanCompilerArgs, + commonLinuxLinkerArgs + asanLinkerArgs, + asanEnv + +// MacOS +addBuildConfiguration 'release', 'macos', 'arm64', + ['-O3', '-DNDEBUG'], + [] +addBuildConfiguration 'debug', 'macos', 'arm64', + ['-O0', '-g', '-DDEBUG'], + [] \ No newline at end of file From 8fa4dd6ee7cb7b7eb98d6e6e2d43d7bc79af733f Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Sat, 25 May 2024 19:22:14 +0200 Subject: [PATCH 13/80] Change asan location command --- gradle/configurations.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index 5a195df45..2468064c1 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -18,7 +18,8 @@ def buildConfigNames() { def locateLibasan() { if (os().isLinux()) { try { - def locateCommand = 'locate libasan.so' + // alternatively we could use `locate libasan.so` but it requires `locate` installed + def locateCommand = 'gcc -print-file-name=libasan.so' def process = locateCommand.execute() process.waitFor() From 778675a106d4da0cb724b2bb57b4ddf694105220 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Sat, 25 May 2024 19:23:01 +0200 Subject: [PATCH 14/80] Remove debug output --- ddprof-test/build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index 186750c12..663966155 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -49,9 +49,7 @@ buildConfigurations.each { config -> if (!config.testEnv.empty) { - println "applying env for test task test${capitalizeFirstLetter(name)}" config.testEnv.each { key, value -> - println "setting env $key=$value for test task test${capitalizeFirstLetter(name)}" environment key, value } } From 845c8896391acd94e18353a142dbe82dfd949b24 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Sat, 25 May 2024 19:44:45 +0200 Subject: [PATCH 15/80] Fix the linux build --- ddprof-lib/build.gradle | 50 ++++----------- gradle/configurations.gradle | 121 ++++++++++++++++++----------------- 2 files changed, 76 insertions(+), 95 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 3fef0d5ba..fd5c608ca 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -24,26 +24,23 @@ def libraryTargetBase(type) { return "${projectDir}/build/native/${type}" } -def libraryTargetPath(type) { - def osarchext = "" +def osarchext() { if (osIdentifier() == 'linux' && archIdentifier() != 'x64') { // when built on aarch64 the location the library is built in is 'x86-64' ¯\_(ツ)_/¯ - osarchext = "x86-64" + return "x86-64" } else if (osIdentifier() == 'macos') { - osarchext = archIdentifier() == 'x64' ? 'x86-64' : 'arm64' + return archIdentifier() == 'x64' ? 'x86-64' : 'arm64' + } else { + return archIdentifier() } - return "${libraryTargetBase(type)}/META-INF/native-libs/${osIdentifier()}-${osarchext}${isMusl() ? '-musl' : ''}" +} + +def libraryTargetPath(type) { + return "${libraryTargetBase(type)}/META-INF/native-libs/${osIdentifier()}-${osarchext()}${isMusl() ? '-musl' : ''}" } def librarySourcePath(type, qualifier = "") { - def osarchext = "" - if (osIdentifier() == 'linux' && archIdentifier() != 'x64') { - // when built on aarch64 the location the library is built in is 'x86-64' ¯\_(ツ)_/¯ - osarchext = "x86-64" - } else if (osIdentifier() == 'macos') { - osarchext = archIdentifier() == 'x64' ? 'x86-64' : 'arm64' - } - return "${projectDir}/build/lib/main/${type}/${osIdentifier()}/${osarchext}/${qualifier}/libjavaProfiler.${osIdentifier() == 'macos' ? 'dylib' : 'so'}" + return "${projectDir}/build/lib/main/${type}/${osIdentifier()}/${osarchext()}/${qualifier}/libjavaProfiler.${osIdentifier() == 'macos' ? 'dylib' : 'so'}" } ext { @@ -194,23 +191,6 @@ gradle.projectsEvaluated { } } } -// // We need to introduce dependencies between generated linker and copy tasks -// // Find all tasks named 'copy*Libs' -// def copyLibsTasks = tasks.matching { -// it.name.startsWith('copy') && it.name.endsWith('Libs') && !it.name.contains('Release') -// } -// // Find all tasks named 'linkLib*' -// def linkLibTasks = tasks.matching { it.name == 'linkLib') } -// -// // For each 'copy*Libs' task, set dependencies on 'linkLib*' tasks -// copyLibsTasks.all { copyTask -> -// linkLibTasks.matching { -// -// }.all { linkTask -> -// copyTask.dependsOn(linkTask) -// println "Configured ${copyTask.name} to depend on ${linkTask.name}" -// } -// } } // configure the compiler here @@ -221,19 +201,16 @@ tasks.withType(CppCompile).configureEach { } } else { onlyIf { - // enable the built-in compiler task only for release; required by the stripsymbols task -// name.contains("Release") + // disable the built-in compiler task for release; we are using the custom compiler task false } } def taskIncludes = ["${javaHome()}/include"] - def taskArgs = ["-fno-omit-frame-pointer", "-momit-leaf-frame-pointer", "-fvisibility=hidden", - "-fdata-sections", "-ffunction-sections", "-std=c++11", "-DPROFILER_VERSION=\"${version}\"", "-DCOUNTERS"] + def taskArgs = [] if (os().isMacOsX()) { taskIncludes.add "${javaHome()}/include/darwin" - taskArgs.addAll(["-D_XOPEN_SOURCE", "-D_DARWIN_C_SOURCE"]) } else if (os().isLinux()) { taskIncludes.add "${javaHome()}/include/linux" if (isMusl()) { @@ -261,8 +238,7 @@ tasks.withType(LinkSharedLibrary).configureEach { } } else { onlyIf { - // enable the built-in linker task only for release; required by the stripsymbols task -// name.contains("Release") + // disable the built-in linker task for release; we are using the custom linker task false } } diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index 2468064c1..7ee8433d2 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -57,79 +57,84 @@ ext { // ======= Define build configurations below ======== +def commonLinuxCompilerArgs = ["-fPIC", "-fno-omit-frame-pointer", "-momit-leaf-frame-pointer", "-fvisibility=hidden", + "-fdata-sections", "-ffunction-sections", "-std=c++11", "-DPROFILER_VERSION=\"${version}\"", "-DCOUNTERS"] + def commonLinuxLinkerArgs = ["-ldl", "-Wl,-z,defs", "--verbose", "-lpthread", "-lm", "-lrt", "-v"] +def commonMacosCompilerArgs = ["-D_XOPEN_SOURCE", "-D_DARWIN_C_SOURCE"] + def asanEnv = libasan != null ? - ['LD_PRELOAD': libasan, - // warning: stack use after return can cause slowness on arm64 - "ASAN_OPTIONS" : "detect_stack_use_after_return=1 suppressions=${projectDir}/asan.supp", - "UBSAN_OPTIONS" : "halt_on_error=1 abort_on_error=1 print_stacktrace=1 suppressions=${projectDir}/ubsan.supp", - // lsan still does not run for all tests - manually trigger on some tests - "LSAN_OPTIONS" : "detect_leaks=0" - ] : [:] + ['LD_PRELOAD': libasan, + // warning: stack use after return can cause slowness on arm64 + "ASAN_OPTIONS" : "detect_stack_use_after_return=1 suppressions=${projectDir}/asan.supp", + "UBSAN_OPTIONS" : "halt_on_error=1 abort_on_error=1 print_stacktrace=1 suppressions=${projectDir}/ubsan.supp", + // lsan still does not run for all tests - manually trigger on some tests + "LSAN_OPTIONS" : "detect_leaks=0" + ] : [:] def asanCompilerArgs = [ - '-g', - // Generate debug information - '-DDEBUG', - // Enable debug mode - '-fPIC', - // Position-independent code - '-fsanitize=address', - // AddressSanitizer for memory errors - '-fsanitize=undefined', - // UndefinedBehaviorSanitizer for undefined behavior - '-fno-sanitize-recover=all', - // Disable recovery to ensure program halts on error - '-fsanitize=float-divide-by-zero', - // Catch floating point division by zero - '-fstack-protector-all', - // Protect against stack buffer overflows - '-fsanitize=leak', - // LeakSanitizer to detect memory leaks - '-fsanitize=pointer-overflow', - // Catch pointer overflows - '-fsanitize=return', - // Check for uninitialized return values - '-fsanitize=bounds', - // BoundsSanitizer for array bounds - '-fsanitize=alignment', - // Catch misaligned pointer access - '-fsanitize=object-size', - // Catch accesses that are likely out of bounds - '-fno-omit-frame-pointer', - // Keep frame pointer for better debugging - '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces + '-g', + // Generate debug information + '-DDEBUG', + // Enable debug mode + '-fPIC', + // Position-independent code + '-fsanitize=address', + // AddressSanitizer for memory errors + '-fsanitize=undefined', + // UndefinedBehaviorSanitizer for undefined behavior + '-fno-sanitize-recover=all', + // Disable recovery to ensure program halts on error + '-fsanitize=float-divide-by-zero', + // Catch floating point division by zero + '-fstack-protector-all', + // Protect against stack buffer overflows + '-fsanitize=leak', + // LeakSanitizer to detect memory leaks + '-fsanitize=pointer-overflow', + // Catch pointer overflows + '-fsanitize=return', + // Check for uninitialized return values + '-fsanitize=bounds', + // BoundsSanitizer for array bounds + '-fsanitize=alignment', + // Catch misaligned pointer access + '-fsanitize=object-size', + // Catch accesses that are likely out of bounds + '-fno-omit-frame-pointer', + // Keep frame pointer for better debugging + '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces ] def asanLinkerArgs = [ - "-fsanitize=address", - "-fsanitize=undefined", - "-fno-omit-frame-pointer" // Often recommended with sanitizers + "-fsanitize=address", + "-fsanitize=undefined", + "-fno-omit-frame-pointer" // Often recommended with sanitizers ] // Linux addBuildConfiguration 'release', 'linux', 'x64', - ['-O3', '-DNDEBUG'], - commonLinuxLinkerArgs + [ - "-Wl,-z,nodelete", - "-static-libstdc++", - "-static-libgcc", - "-Wl,--exclude-libs,ALL", - "-Wl,--gc-sections" - ] + ['-O3', '-DNDEBUG'] + commonLinuxCompilerArgs, + commonLinuxLinkerArgs + [ + "-Wl,-z,nodelete", + "-static-libstdc++", + "-static-libgcc", + "-Wl,--exclude-libs,ALL", + "-Wl,--gc-sections" + ] addBuildConfiguration 'debug', 'linux', 'x64', - ['-O0', '-g', '-DDEBUG'], - commonLinuxLinkerArgs + ['-O0', '-g', '-DDEBUG'] + commonLinuxCompilerArgs, + commonLinuxLinkerArgs addBuildConfiguration 'asan', 'linux', 'x64', - asanCompilerArgs, - commonLinuxLinkerArgs + asanLinkerArgs, - asanEnv + asanCompilerArgs, + commonLinuxLinkerArgs + asanLinkerArgs, + asanEnv // MacOS addBuildConfiguration 'release', 'macos', 'arm64', - ['-O3', '-DNDEBUG'], - [] + ['-O3', '-DNDEBUG'], + [] addBuildConfiguration 'debug', 'macos', 'arm64', - ['-O0', '-g', '-DDEBUG'], - [] \ No newline at end of file + ['-O0', '-g', '-DDEBUG'], + [] \ No newline at end of file From 8e1fa055037a3c2451adc13975c0264aa0c04940 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Mon, 27 May 2024 11:56:17 +0200 Subject: [PATCH 16/80] Add a tsan configuration --- ddprof-test/build.gradle | 10 +++++ ddprof-test/tsan.supp | 0 gradle/configurations.gradle | 79 ++++++++++++++++++++++++++---------- 3 files changed, 68 insertions(+), 21 deletions(-) create mode 100644 ddprof-test/tsan.supp diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index 663966155..81d84618a 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -30,6 +30,8 @@ buildConfigurations.each { config -> if (config.os != osIdentifier() || config.arch != archIdentifier()) { return } + logger.lifecycle("Creating configuration for ${name}") + def cfg = configurations.create("test${capitalizeFirstLetter(name)}Implementation") { canBeConsumed = true canBeResolved = true @@ -52,6 +54,14 @@ buildConfigurations.each { config -> config.testEnv.each { key, value -> environment key, value } + logger.lifecycle("Setting environment variables for ${name}: ${config.testEnv}") + } + // Exclude patterns for ASAN and TSAN tests + if (name.contains('asan')) { + exclude '**/endpoints*' + } + if (name.contains('tsan')) { + exclude '**/cpu*' } } if (name == 'debug') { diff --git a/ddprof-test/tsan.supp b/ddprof-test/tsan.supp new file mode 100644 index 000000000..e69de29bb diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index 7ee8433d2..c773ffe05 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -15,32 +15,39 @@ def buildConfigNames() { }.collect { it.name }.toSet() } -def locateLibasan() { - if (os().isLinux()) { - try { - // alternatively we could use `locate libasan.so` but it requires `locate` installed - def locateCommand = 'gcc -print-file-name=libasan.so' - def process = locateCommand.execute() - process.waitFor() - - if (process.exitValue() == 0) { - def output = process.in.text - def lines = output.readLines() - def exactMatch = lines.find { it ==~ /.*libasan\.so$/ } - - if (exactMatch) { - return exactMatch +def locateLibrary(String libName) { + if (os().isLinux()) { + try { + def locateCommand = "gcc -print-file-name=${libName}.so" + def process = locateCommand.execute() + process.waitFor() + + if (process.exitValue() == 0) { + def output = process.in.text.trim() + if (output.endsWith("${libName}.so")) { + return output + } + } + } catch (Exception e) { + log.lifecycle("Exception when locating ${libName} library: ${e.message}") } - } - } catch (Throwable t) { - // ignored } - } - return null + return null +} + +def locateLibasan() { + return locateLibrary('libasan') +} + +def locateLibtsan() { + return locateLibrary('libtsan') } def libasan = locateLibasan() +def libtsan = locateLibtsan() +logger.lifecycle("Found ASAN library: ${libasan}") +logger.lifecycle("Found TSAN library: ${libtsan}") def hasAsan() { return libasan != null } @@ -113,6 +120,30 @@ def asanLinkerArgs = [ "-fno-omit-frame-pointer" // Often recommended with sanitizers ] +def tsanCompilerArgs = [ + '-g', + // Generate debug information + '-DDEBUG', + // Enable debug mode + '-fPIC', + // Position-independent code + '-fsanitize=thread', + // ThreadSanitizer for + '-fno-omit-frame-pointer', + // Keep frame pointer for better debugging + '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces +] + +def tsanLinkerArgs = [ + "-fsanitize=thread", + "-fno-omit-frame-pointer" // Often recommended with sanitizers +] + +def tsanEnv = libtsan != null ? + ['LD_PRELOAD': libtsan, + "TSAN_OPTIONS" : "suppressions=${projectDir}/tsan.supp" + ] : [:] + // Linux addBuildConfiguration 'release', 'linux', 'x64', ['-O3', '-DNDEBUG'] + commonLinuxCompilerArgs, @@ -126,11 +157,17 @@ addBuildConfiguration 'release', 'linux', 'x64', addBuildConfiguration 'debug', 'linux', 'x64', ['-O0', '-g', '-DDEBUG'] + commonLinuxCompilerArgs, commonLinuxLinkerArgs + addBuildConfiguration 'asan', 'linux', 'x64', - asanCompilerArgs, + asanCompilerArgs + commonLinuxCompilerArgs, commonLinuxLinkerArgs + asanLinkerArgs, asanEnv +addBuildConfiguration 'tsan', 'linux', 'x64', + tsanCompilerArgs + commonLinuxCompilerArgs, + commonLinuxLinkerArgs + tsanLinkerArgs, + tsanEnv + // MacOS addBuildConfiguration 'release', 'macos', 'arm64', ['-O3', '-DNDEBUG'], From 6d03be8821b5bf3016787f8073c28aed2c09cf53 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Mon, 27 May 2024 13:55:12 +0200 Subject: [PATCH 17/80] Minor tsan fix - Ensure we use atomic primitives on the epoch value. --- ddprof-lib/src/main/cpp/profiler.cpp | 4 ++-- ddprof-lib/src/main/cpp/profiler.h | 3 ++- ddprof-test/asan.supp | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index b077ea29c..a72e5fda2 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -1143,7 +1143,7 @@ Error Profiler::start(Arguments& args, bool reset) { _state = RUNNING; _start_time = time(NULL); - _epoch++; + __atomic_add_fetch(&_epoch, 1, __ATOMIC_SEQ_CST); return Error::OK; } @@ -1256,7 +1256,7 @@ Error Profiler::dump(const char* path, const int length) { lockAll(); Error err = _jfr.dump(path, length); - _epoch++; + __atomic_add_fetch(&_epoch, 1, __ATOMIC_SEQ_CST); // Reset calltrace storage if (!_omit_stacktraces) { diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 3f4b77f17..12bbd5c93 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -253,7 +253,8 @@ class Profiler { } inline u32 recordingEpoch() { - return _epoch; + // no thread reordering constraints + return __atomic_load_n(&_epoch, __ATOMIC_RELAXED); } Error run(Arguments& args); diff --git a/ddprof-test/asan.supp b/ddprof-test/asan.supp index 7041f6237..e69de29bb 100644 --- a/ddprof-test/asan.supp +++ b/ddprof-test/asan.supp @@ -1 +0,0 @@ -interceptor_via_fun:get16 From 9559071e09b40f8fdaad907c00dc3a529fb6b2f0 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Mon, 27 May 2024 14:13:13 +0200 Subject: [PATCH 18/80] TSan fix - Call Trace Storage Ensure we load the value using an atomic primitive when accessing the call trace storage --- ddprof-lib/src/main/cpp/callTraceStorage.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ddprof-lib/src/main/cpp/callTraceStorage.cpp b/ddprof-lib/src/main/cpp/callTraceStorage.cpp index f5c060e7f..5ecec91cf 100644 --- a/ddprof-lib/src/main/cpp/callTraceStorage.cpp +++ b/ddprof-lib/src/main/cpp/callTraceStorage.cpp @@ -227,12 +227,15 @@ u32 CallTraceStorage::put(int num_frames, ASGCT_CallFrame* frames, bool truncate u32 slot = hash & (capacity - 1); u32 step = 0; - while (keys[slot] != hash) { - if (keys[slot] == 0) { + while (true) { + int key_value = __atomic_load_n(&keys[slot], __ATOMIC_RELAXED); + if (key_value == hash) { // Hash matches, exit the loop + break; + } + if (key_value == 0) { if (!__sync_bool_compare_and_swap(&keys[slot], 0, hash)) { - continue; + continue; // another thread claimed it, go to next slot } - // Increment the table size, and if the load factor exceeds 0.75, reserve a new table if (table->incSize() == capacity * 3 / 4) { LongHashTable* new_table = LongHashTable::allocate(table, capacity * 2); From 77ccc3f43bc28fddeaa78c299c3ad230669b4861 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Mon, 27 May 2024 15:02:36 +0200 Subject: [PATCH 19/80] TSan: WallClock running flag - Minor refactor to remove the volatile keyword - Added some atomic primitives to avoid a TSan warning on the read of the value --- ddprof-lib/src/main/cpp/wallClock.cpp | 10 +++++++--- ddprof-lib/src/main/cpp/wallClock.h | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ddprof-lib/src/main/cpp/wallClock.cpp b/ddprof-lib/src/main/cpp/wallClock.cpp index db5e9a272..5a9e37bec 100644 --- a/ddprof-lib/src/main/cpp/wallClock.cpp +++ b/ddprof-lib/src/main/cpp/wallClock.cpp @@ -125,9 +125,13 @@ Error WallClock::start(Arguments &args) { } void WallClock::stop() { - _running = false; + _running.store(false); + // the thread join ensures we wait for the thread to finish before returning (and possibly removing the object) pthread_kill(_thread, WAKEUP_SIGNAL); - pthread_join(_thread, NULL); + int res = pthread_join(_thread, NULL); + if(res != 0) { + Log::warn("Unable to join WallClock thread on stop %d", res); + } } void WallClock::timerLoop() { @@ -149,7 +153,7 @@ void WallClock::timerLoop() { u64 startTime = TSC::ticks(); WallClockEpochEvent epoch(startTime); - while (_running) { + while (_running.load(std::memory_order_relaxed)) { if (thread_filter->enabled()) { thread_filter->collect(tids); } else { diff --git a/ddprof-lib/src/main/cpp/wallClock.h b/ddprof-lib/src/main/cpp/wallClock.h index 405056861..81263a069 100644 --- a/ddprof-lib/src/main/cpp/wallClock.h +++ b/ddprof-lib/src/main/cpp/wallClock.h @@ -17,6 +17,7 @@ #ifndef _WALLCLOCK_H #define _WALLCLOCK_H +#include #include #include #include @@ -36,7 +37,7 @@ class WallClock : public Engine { // to avoid contention on a spin lock inside Profiler::recordSample(). int _reservoir_size; - volatile bool _running; + std::atomic _running; pthread_t _thread; void timerLoop(); From 384b379a7ff6e987c07572d1285dbbbba24d3c15 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Mon, 27 May 2024 15:29:32 +0200 Subject: [PATCH 20/80] - Add suppressions for java issues - Minor fix on thread filter APIs --- ddprof-lib/src/main/cpp/threadFilter.h | 3 ++- ddprof-test/tsan.supp | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/threadFilter.h b/ddprof-lib/src/main/cpp/threadFilter.h index 099459331..40e2d7c42 100644 --- a/ddprof-lib/src/main/cpp/threadFilter.h +++ b/ddprof-lib/src/main/cpp/threadFilter.h @@ -40,10 +40,11 @@ class ThreadFilter { if (thread_id >= _max_thread_id) { return NULL; } - return _bitmap[(u32)thread_id / BITMAP_CAPACITY]; + return __atomic_load_n(&(_bitmap[static_cast(thread_id) / BITMAP_CAPACITY]), __ATOMIC_ACQUIRE); } u64& word(u64* bitmap, int thread_id) { + // todo: add thread safe APIs return bitmap[((u32)thread_id % BITMAP_CAPACITY) >> 6]; } diff --git a/ddprof-test/tsan.supp b/ddprof-test/tsan.supp index e69de29bb..20ef8dd73 100644 --- a/ddprof-test/tsan.supp +++ b/ddprof-test/tsan.supp @@ -0,0 +1,10 @@ +# Suppress data race in libzip.so +race:libzip.so +# Suppress data race in G1Allocator::survivor_attempt_allocation +race:G1Allocator::survivor_attempt_allocation +race:G1CollectedHeap::allocate_new_tlab +race:OopMapCache::lookup +race:G1CodeRootSet::clear +race:Metaspace::allocate +race:FilterQueue +race:PlatformParker::~PlatformParker From 2b90274587bf04352748eef83b6d122441220b2f Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 27 May 2024 15:31:59 +0200 Subject: [PATCH 21/80] Move gtest to gradle --- ddprof-lib/build.gradle | 76 +++++++++--- ddprof-lib/src/test/run_tests.sh | 28 +++++ ddprof-lib/test.sh | 34 ----- gradle/configurations.gradle | 207 ++++++++++++++++--------------- 4 files changed, 192 insertions(+), 153 deletions(-) create mode 100755 ddprof-lib/src/test/run_tests.sh delete mode 100755 ddprof-lib/test.sh diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index fd5c608ca..88d7581fe 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -6,6 +6,7 @@ plugins { id 'com.github.ben-manes.versions' version '0.27.0' id "com.diffplug.spotless" version "6.11.0" + id 'de.undercouch.download' version '4.1.1' } def libraryName = "ddprof" @@ -53,6 +54,40 @@ sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 def isGitlabCI = System.getenv("GITLAB_CI") != null +def buildTempDir = "${projectDir}/build/tmp" +def gtestVersion = "release-1.12.1" +def gtestDir = "${buildTempDir}/gtest" +def gtestLib = "${gtestDir}/build/lib/libgtest.a" + +tasks.register('downloadGTest', Download) { + src "https://github.com/google/googletest/archive/refs/tags/${gtestVersion}.zip" + dest "${projectDir}/build/tmp/gtest.zip" + overwrite false +} + +tasks.register('unzipGTest', Copy) { + dependsOn downloadGTest + from zipTree(tasks.downloadGTest.dest) + into "${gtestDir}" + eachFile { FileCopyDetails details -> + def pathSegments = details.relativePath.segments + if (pathSegments.length > 1) { + details.relativePath = new RelativePath(true, *pathSegments.drop(1)) + } else { + details.exclude() + } + } + doLast { + // take care of the dangling original top-level dir + file("${projectDir}/build/tmp/gtest/googletest-${gtestVersion}").deleteDir() + } +} + +tasks.register('prepareGTest', Exec) { + dependsOn unzipGTest + workingDir gtestDir + commandLine "sh", "-c", "cmake . -DCMAKE_CXX_STANDARD=11 -DCMAKE_OSX_DEPLOYMENT_TARGET=10.10 && make" +} // Allow specifying the external location for the native libraries // The libraries should be properly sorted into subfolders corresponding to the `libraryTargetPath` value for each @@ -67,18 +102,6 @@ tasks.register('copyExternalLibs', Copy) { } } -// the bridge between the gradle build and gtests -// the gradle gtest plugin(s) are quite unusable at this moment - using a shell script seems to be the only sane option -tasks.register('cppTest', Exec) { - if (project.hasProperty('skip-tests') || project.hasProperty('skip-cpp-tests')) { - environment 'SKIP_TESTS', 'true' - } - workingDir "$projectDir" - commandLine './test.sh' - - dependsOn assemble -} - tasks.register('assembleAll') {} // use the build config names to create configurations, copy lib and asemble jar tasks @@ -89,12 +112,19 @@ buildConfigNames().each { name -> canBeResolved = false extendsFrom configurations.implementation } + + def gtestTask = tasks.register("gtest${capitalizeFirstLetter(name)}", Exec) { + onlyIf { + !project.hasProperty('skip-tests') && !project.hasProperty('skip-cpp-tests') + } + dependsOn prepareGTest + workingDir projectDir + commandLine "./src/test/run_tests.sh", projectDir, name, gtestDir + } def copyTask = tasks.register("copy${capitalizeFirstLetter(name)}Libs", Copy) { - dependsOn cppTest from file(librarySourcePath(name, name == 'release' ? 'stripped' : '')).parent // the release build is stripped into file(libraryTargetPath(name)) } - def assembleTask = tasks.register("assemble${capitalizeFirstLetter(name)}Jar", Jar) { group = 'build' description = "Assemble the ${name} build of the library" @@ -183,12 +213,20 @@ gradle.projectsEvaluated { tasks.copyReleaseLibs.dependsOn tasks.stripLibRelease buildConfigNames().each { - if (it != 'release') { - def linkTask = tasks.findByName("linkLib${capitalizeFirstLetter(it)}") - def copyTask = tasks.findByName("copy${capitalizeFirstLetter(it)}Libs") - if (linkTask != null) { - copyTask.dependsOn linkTask + def compileTask = tasks.findByName("compileLib${capitalizeFirstLetter(it)}") + def gtestTask = tasks.findByName("gtest${capitalizeFirstLetter(it)}") + if (compileTask != null && gtestTask != null) { + gtestTask.dependsOn compileTask + } + def linkTask = tasks.findByName("linkLib${capitalizeFirstLetter(it)}") + if (linkTask != null) { + if (it != 'release') { + def copyTask = tasks.findByName("copy${capitalizeFirstLetter(it)}Libs") + if (copyTask != null) { + copyTask.dependsOn linkTask + } } + linkTask.dependsOn gtestTask } } } diff --git a/ddprof-lib/src/test/run_tests.sh b/ddprof-lib/src/test/run_tests.sh new file mode 100755 index 000000000..48c7ee7e7 --- /dev/null +++ b/ddprof-lib/src/test/run_tests.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +PROJECT_DIR=$1 +CONFIG_NAME=$2 +GTEST_DIR=$3 + +function build_test() { + NAME=$1 + echo "Building test ${NAME}..." + # shellcheck disable=SC2038 + find ${PROJECT_DIR}/build/obj/main/${CONFIG_NAME} -name "*.o" | + xargs g++ -std=c++11 -o ${TMPDIR}/${NAME} ${PROJECT_DIR}/src/test/cpp/${NAME}.cpp \ + -I${GTEST_DIR}/googletest/include -I${GTEST_DIR}/googlemock/include \ + -I${PROJECT_DIR}/src/main/cpp ${GTEST_DIR}/lib/libgtest.a \ + ${GTEST_DIR}/lib/libgtest_main.a ${GTEST_DIR}/lib/libgmock.a \ + ${GTEST_DIR}/lib/libgmock_main.a -pthread +} + +function build_and_run() { + NAME=$1 + build_test ${NAME} + echo "Running test ${NAME}..." + ${TMPDIR}/${NAME} +} + +build_and_run ddprof_ut +build_and_run demangle_ut + diff --git a/ddprof-lib/test.sh b/ddprof-lib/test.sh deleted file mode 100755 index 50a1922dc..000000000 --- a/ddprof-lib/test.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -set -e - -HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -if [ ! -z "$SKIP_TESTS" ]; then - echo "Skipping CPP tests" - exit 0 -fi - -CMAKE=$(which cmake) -if [ -z "$CMAKE" ]; then - echo "[ERROR] Please, install cmake" - exit 1 -fi - - -function build_and_test() { - BUILD_TYPE=$1 - TARGET=${HERE}/build_${BUILD_TYPE} - cd ${HERE}/src/test - cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -Wno-dev -S . -B ${TARGET} - cmake --build ${TARGET} - cd ${TARGET} && ctest -} - -if [ -z "${BUILD_TYPE}" ]; then - BUILD_TYPE="Debug" -fi - -# disable core files -ulimit -c 0 -build_and_test ${BUILD_TYPE} diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index c773ffe05..e4873b695 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -16,38 +16,38 @@ def buildConfigNames() { } def locateLibrary(String libName) { - if (os().isLinux()) { - try { - def locateCommand = "gcc -print-file-name=${libName}.so" - def process = locateCommand.execute() - process.waitFor() - - if (process.exitValue() == 0) { - def output = process.in.text.trim() - if (output.endsWith("${libName}.so")) { - return output - } - } - } catch (Exception e) { - log.lifecycle("Exception when locating ${libName} library: ${e.message}") + if (os().isLinux()) { + try { + def locateCommand = "gcc -print-file-name=${libName}.so" + def process = locateCommand.execute() + process.waitFor() + + if (process.exitValue() == 0) { + def output = process.in.text.trim() + if (output.endsWith("${libName}.so")) { + return output } + } + } catch (Exception e) { + log.lifecycle("Exception when locating ${libName} library: ${e.message}") } - return null + } + return null } def locateLibasan() { - return locateLibrary('libasan') + return locateLibrary('libasan') } def locateLibtsan() { - return locateLibrary('libtsan') + return locateLibrary('libtsan') } def libasan = locateLibasan() def libtsan = locateLibtsan() -logger.lifecycle("Found ASAN library: ${libasan}") -logger.lifecycle("Found TSAN library: ${libtsan}") +//logger.lifecycle("Found ASAN library: ${libasan}") +//logger.lifecycle("Found TSAN library: ${libtsan}") def hasAsan() { return libasan != null } @@ -64,114 +64,121 @@ ext { // ======= Define build configurations below ======== -def commonLinuxCompilerArgs = ["-fPIC", "-fno-omit-frame-pointer", "-momit-leaf-frame-pointer", "-fvisibility=hidden", - "-fdata-sections", "-ffunction-sections", "-std=c++11", "-DPROFILER_VERSION=\"${version}\"", "-DCOUNTERS"] +def commonLinuxCompilerArgs = [ + "-fPIC", + "-fno-omit-frame-pointer", + "-momit-leaf-frame-pointer", + "-fvisibility=hidden", + "-fdata-sections", + "-ffunction-sections", + "-std=c++11", + "-DPROFILER_VERSION=\"${version}\"", + "-DCOUNTERS" +] def commonLinuxLinkerArgs = ["-ldl", "-Wl,-z,defs", "--verbose", "-lpthread", "-lm", "-lrt", "-v"] -def commonMacosCompilerArgs = ["-D_XOPEN_SOURCE", "-D_DARWIN_C_SOURCE"] +def commonMacosCompilerArgs = commonLinuxCompilerArgs + ["-D_XOPEN_SOURCE", "-D_DARWIN_C_SOURCE"] def asanEnv = libasan != null ? - ['LD_PRELOAD': libasan, - // warning: stack use after return can cause slowness on arm64 - "ASAN_OPTIONS" : "detect_stack_use_after_return=1 suppressions=${projectDir}/asan.supp", - "UBSAN_OPTIONS" : "halt_on_error=1 abort_on_error=1 print_stacktrace=1 suppressions=${projectDir}/ubsan.supp", - // lsan still does not run for all tests - manually trigger on some tests - "LSAN_OPTIONS" : "detect_leaks=0" - ] : [:] + ['LD_PRELOAD': libasan, + // warning: stack use after return can cause slowness on arm64 + "ASAN_OPTIONS" : "detect_stack_use_after_return=1 suppressions=${projectDir}/asan.supp", + "UBSAN_OPTIONS" : "halt_on_error=1 abort_on_error=1 print_stacktrace=1 suppressions=${projectDir}/ubsan.supp", + // lsan still does not run for all tests - manually trigger on some tests + "LSAN_OPTIONS" : "detect_leaks=0" + ] : [:] def asanCompilerArgs = [ - '-g', - // Generate debug information - '-DDEBUG', - // Enable debug mode - '-fPIC', - // Position-independent code - '-fsanitize=address', - // AddressSanitizer for memory errors - '-fsanitize=undefined', - // UndefinedBehaviorSanitizer for undefined behavior - '-fno-sanitize-recover=all', - // Disable recovery to ensure program halts on error - '-fsanitize=float-divide-by-zero', - // Catch floating point division by zero - '-fstack-protector-all', - // Protect against stack buffer overflows - '-fsanitize=leak', - // LeakSanitizer to detect memory leaks - '-fsanitize=pointer-overflow', - // Catch pointer overflows - '-fsanitize=return', - // Check for uninitialized return values - '-fsanitize=bounds', - // BoundsSanitizer for array bounds - '-fsanitize=alignment', - // Catch misaligned pointer access - '-fsanitize=object-size', - // Catch accesses that are likely out of bounds - '-fno-omit-frame-pointer', - // Keep frame pointer for better debugging - '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces + '-g', + // Generate debug information + '-DDEBUG', + // Enable debug mode + '-fPIC', + // Position-independent code + '-fsanitize=address', + // AddressSanitizer for memory errors + '-fsanitize=undefined', + // UndefinedBehaviorSanitizer for undefined behavior + '-fno-sanitize-recover=all', + // Disable recovery to ensure program halts on error + '-fsanitize=float-divide-by-zero', + // Catch floating point division by zero + '-fstack-protector-all', + // Protect against stack buffer overflows + '-fsanitize=leak', + // LeakSanitizer to detect memory leaks + '-fsanitize=pointer-overflow', + // Catch pointer overflows + '-fsanitize=return', + // Check for uninitialized return values + '-fsanitize=bounds', + // BoundsSanitizer for array bounds + '-fsanitize=alignment', + // Catch misaligned pointer access + '-fsanitize=object-size', + // Catch accesses that are likely out of bounds + '-fno-omit-frame-pointer', + // Keep frame pointer for better debugging + '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces ] def asanLinkerArgs = [ - "-fsanitize=address", - "-fsanitize=undefined", - "-fno-omit-frame-pointer" // Often recommended with sanitizers + "-fsanitize=address", + "-fsanitize=undefined", + "-fno-omit-frame-pointer" // Often recommended with sanitizers ] def tsanCompilerArgs = [ - '-g', - // Generate debug information - '-DDEBUG', - // Enable debug mode - '-fPIC', - // Position-independent code - '-fsanitize=thread', - // ThreadSanitizer for - '-fno-omit-frame-pointer', - // Keep frame pointer for better debugging - '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces + '-g', + // Generate debug information + '-DDEBUG', + // Enable debug mode + '-fPIC', + // Position-independent code + '-fsanitize=thread', + // ThreadSanitizer for + '-fno-omit-frame-pointer', + // Keep frame pointer for better debugging + '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces ] -def tsanLinkerArgs = [ - "-fsanitize=thread", - "-fno-omit-frame-pointer" // Often recommended with sanitizers +def tsanLinkerArgs = ["-fsanitize=thread", "-fno-omit-frame-pointer" // Often recommended with sanitizers ] def tsanEnv = libtsan != null ? - ['LD_PRELOAD': libtsan, - "TSAN_OPTIONS" : "suppressions=${projectDir}/tsan.supp" - ] : [:] + ['LD_PRELOAD': libtsan, + "TSAN_OPTIONS" : "suppressions=${projectDir}/tsan.supp" + ] : [:] // Linux addBuildConfiguration 'release', 'linux', 'x64', - ['-O3', '-DNDEBUG'] + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + [ - "-Wl,-z,nodelete", - "-static-libstdc++", - "-static-libgcc", - "-Wl,--exclude-libs,ALL", - "-Wl,--gc-sections" - ] + ['-O3', '-DNDEBUG'] + commonLinuxCompilerArgs, + commonLinuxLinkerArgs + [ + "-Wl,-z,nodelete", + "-static-libstdc++", + "-static-libgcc", + "-Wl,--exclude-libs,ALL", + "-Wl,--gc-sections" + ] addBuildConfiguration 'debug', 'linux', 'x64', - ['-O0', '-g', '-DDEBUG'] + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + ['-O0', '-g', '-DDEBUG'] + commonLinuxCompilerArgs, + commonLinuxLinkerArgs addBuildConfiguration 'asan', 'linux', 'x64', - asanCompilerArgs + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + asanLinkerArgs, - asanEnv + asanCompilerArgs + commonLinuxCompilerArgs, + commonLinuxLinkerArgs + asanLinkerArgs, + asanEnv addBuildConfiguration 'tsan', 'linux', 'x64', - tsanCompilerArgs + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + tsanLinkerArgs, - tsanEnv + tsanCompilerArgs + commonLinuxCompilerArgs, + commonLinuxLinkerArgs + tsanLinkerArgs, + tsanEnv // MacOS addBuildConfiguration 'release', 'macos', 'arm64', - ['-O3', '-DNDEBUG'], - [] + commonMacosCompilerArgs + ['-O3', '-DNDEBUG'], + [] addBuildConfiguration 'debug', 'macos', 'arm64', - ['-O0', '-g', '-DDEBUG'], - [] \ No newline at end of file + commonMacosCompilerArgs + ['-O0', '-g', '-DDEBUG'], + [] \ No newline at end of file From 61d9decbf7a391ed45c479974f70d0ee1ba2fb79 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 27 May 2024 15:37:13 +0200 Subject: [PATCH 22/80] Remove access to non-existent file in CI --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e8f4dcba..fa797be0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,6 @@ jobs: export LIBC=glibc export JAVA_TEST_HOME=$(pwd)/test_jdk export BUILD_TYPE=SanitizedDebug - chmod a+x ddprof-lib/test.sh ./gradlew :ddprof-test:test --info - name: Upload logs uses: actions/upload-artifact@v3 @@ -157,7 +156,6 @@ jobs: export LIBC=glibc export BUILD_TYPE=SanitizedDebug chmod a+x gradlew - chmod a+x ddprof-lib/test.sh ./gradlew :ddprof-test:test --info - name: Upload logs uses: actions/upload-artifact@v3 @@ -255,8 +253,7 @@ jobs: export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=musl/${{ matrix.java_version }} export LIBC=musl - export JAVA_TEST_HOME=$(pwd)/test_jdk - chmod a+x ddprof-lib/test.sh + export JAVA_TEST_HOME=$(pwd)/test_jdkTry ./gradlew :ddprof-test:test --info - name: Upload logs uses: actions/upload-artifact@v3 From 9f1da2e7cf3075a23634d0038325411cbb14fcdc Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 27 May 2024 15:48:45 +0200 Subject: [PATCH 23/80] Debug gtests on CI --- ddprof-lib/src/test/run_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ddprof-lib/src/test/run_tests.sh b/ddprof-lib/src/test/run_tests.sh index 48c7ee7e7..96e288015 100755 --- a/ddprof-lib/src/test/run_tests.sh +++ b/ddprof-lib/src/test/run_tests.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -x + PROJECT_DIR=$1 CONFIG_NAME=$2 GTEST_DIR=$3 From 5b3f463763de7099fcc0d363f995dd05a7af2dc8 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 27 May 2024 16:02:35 +0200 Subject: [PATCH 24/80] TMPDIR fallback --- ddprof-lib/src/test/run_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ddprof-lib/src/test/run_tests.sh b/ddprof-lib/src/test/run_tests.sh index 96e288015..63cf8d6d5 100755 --- a/ddprof-lib/src/test/run_tests.sh +++ b/ddprof-lib/src/test/run_tests.sh @@ -6,6 +6,8 @@ PROJECT_DIR=$1 CONFIG_NAME=$2 GTEST_DIR=$3 +TMPDIR="{${TMPDIR:-$(dirname $(mktemp))/}" + function build_test() { NAME=$1 echo "Building test ${NAME}..." From 2266b46202dac86490950a84fef569579adf5043 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 27 May 2024 16:05:08 +0200 Subject: [PATCH 25/80] TMPDIR fallback (1) --- ddprof-lib/src/test/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddprof-lib/src/test/run_tests.sh b/ddprof-lib/src/test/run_tests.sh index 63cf8d6d5..f4f510a5f 100755 --- a/ddprof-lib/src/test/run_tests.sh +++ b/ddprof-lib/src/test/run_tests.sh @@ -6,7 +6,7 @@ PROJECT_DIR=$1 CONFIG_NAME=$2 GTEST_DIR=$3 -TMPDIR="{${TMPDIR:-$(dirname $(mktemp))/}" +TMPDIR="${TMPDIR:-$(dirname $(mktemp))}" function build_test() { NAME=$1 From 6e9ce04e243a2cafaafeb9b7a1a1726cfc478555 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 27 May 2024 16:07:26 +0200 Subject: [PATCH 26/80] Turn off CI script debug --- ddprof-lib/src/test/run_tests.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ddprof-lib/src/test/run_tests.sh b/ddprof-lib/src/test/run_tests.sh index f4f510a5f..739b5ebb7 100755 --- a/ddprof-lib/src/test/run_tests.sh +++ b/ddprof-lib/src/test/run_tests.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -set -x - PROJECT_DIR=$1 CONFIG_NAME=$2 GTEST_DIR=$3 From dbd3fdfe179777c29d47c9b64561c1bd8538d0bd Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 27 May 2024 16:12:39 +0200 Subject: [PATCH 27/80] Fix codecheck --- .github/workflows/codecheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codecheck.yml b/.github/workflows/codecheck.yml index 57a107885..33fd73edb 100644 --- a/.github/workflows/codecheck.yml +++ b/.github/workflows/codecheck.yml @@ -120,6 +120,6 @@ jobs: # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main - - run: ./gradlew -x test assembleJar + - run: ./gradlew -x test assembleReleaseJar - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 From 1a24bfa5e8f62b06032aaa04eceb2c55cc909f0f Mon Sep 17 00:00:00 2001 From: r1viollet Date: Mon, 27 May 2024 16:30:00 +0200 Subject: [PATCH 28/80] TSan fix - Silence alignment issue from checkInterruptedSyscall --- ddprof-lib/src/main/cpp/stackFrame_x64.cpp | 3 +++ ddprof-lib/src/main/cpp/threadFilter.cpp | 2 ++ ddprof-test/build.gradle | 8 -------- ddprof-test/tsan.supp | 9 +++++++++ gradle/configurations.gradle | 8 ++++++-- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/ddprof-lib/src/main/cpp/stackFrame_x64.cpp b/ddprof-lib/src/main/cpp/stackFrame_x64.cpp index 64e06db13..c059838ce 100644 --- a/ddprof-lib/src/main/cpp/stackFrame_x64.cpp +++ b/ddprof-lib/src/main/cpp/stackFrame_x64.cpp @@ -201,6 +201,9 @@ bool StackFrame::skipFaultInstruction() { return false; } +// when accessing the immediate value to read the syscal value, we do not check for alignment issue +// This can have performance penalties, though it is OK on x86_64 +__attribute__((no_sanitize("address"))) bool StackFrame::checkInterruptedSyscall() { #ifdef __APPLE__ // We are not interested in syscalls that do not check error code, e.g. semaphore_wait_trap diff --git a/ddprof-lib/src/main/cpp/threadFilter.cpp b/ddprof-lib/src/main/cpp/threadFilter.cpp index 7e64ce02b..976bc0ec2 100644 --- a/ddprof-lib/src/main/cpp/threadFilter.cpp +++ b/ddprof-lib/src/main/cpp/threadFilter.cpp @@ -16,6 +16,8 @@ #include #include +#include + #include "threadFilter.h" #include "os.h" #include "counters.h" diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index 81d84618a..048491e75 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -49,20 +49,12 @@ buildConfigurations.each { config -> !file.name.contains('ddprof-') || file.name.contains('test-tracer') } + cfg - if (!config.testEnv.empty) { config.testEnv.each { key, value -> environment key, value } logger.lifecycle("Setting environment variables for ${name}: ${config.testEnv}") } - // Exclude patterns for ASAN and TSAN tests - if (name.contains('asan')) { - exclude '**/endpoints*' - } - if (name.contains('tsan')) { - exclude '**/cpu*' - } } if (name == 'debug') { // redirect the default test task to the debug test task diff --git a/ddprof-test/tsan.supp b/ddprof-test/tsan.supp index 20ef8dd73..e9c7fdb89 100644 --- a/ddprof-test/tsan.supp +++ b/ddprof-test/tsan.supp @@ -3,8 +3,17 @@ race:libzip.so # Suppress data race in G1Allocator::survivor_attempt_allocation race:G1Allocator::survivor_attempt_allocation race:G1CollectedHeap::allocate_new_tlab +race:G1CollectedHeap::mem_allocate race:OopMapCache::lookup race:G1CodeRootSet::clear race:Metaspace::allocate race:FilterQueue race:PlatformParker::~PlatformParker +race:ObjAllocator::initialize +# Todo: jvmti could be interesting +race:JvmtiExport::post_sampled_object_alloc +race:SymbolTable::do_add_if_needed +race:SymbolTable::do_lookup +race:G1ParScanThreadState::copy_to_survivor_space +race:G1RemSetScanState::G1ClearCardTableTask +race:ObjArrayAllocator::initialize diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index c773ffe05..29fd3d253 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -46,8 +46,12 @@ def locateLibtsan() { def libasan = locateLibasan() def libtsan = locateLibtsan() -logger.lifecycle("Found ASAN library: ${libasan}") -logger.lifecycle("Found TSAN library: ${libtsan}") +if (libasan == null) { + logger.lifecycle("Asan library not found") +} +if (libtsan == null) { + logger.lifecycle("Tsan library not found") +} def hasAsan() { return libasan != null } From 45b713fb539ad91a65a001b7fbb78d4c665d45fb Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 27 May 2024 18:01:09 +0000 Subject: [PATCH 29/80] Gtest task per configuration --- ddprof-lib/build.gradle | 50 +++++++++++++++++++--------- ddprof-lib/src/test/prepare_gtest.sh | 12 +++++++ ddprof-lib/src/test/run_tests.sh | 23 ++++++++----- ddprof-test/build.gradle | 1 + gradle/configurations.gradle | 7 ++++ 5 files changed, 68 insertions(+), 25 deletions(-) create mode 100755 ddprof-lib/src/test/prepare_gtest.sh diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 88d7581fe..a9ba37920 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -83,12 +83,6 @@ tasks.register('unzipGTest', Copy) { } } -tasks.register('prepareGTest', Exec) { - dependsOn unzipGTest - workingDir gtestDir - commandLine "sh", "-c", "cmake . -DCMAKE_CXX_STANDARD=11 -DCMAKE_OSX_DEPLOYMENT_TARGET=10.10 && make" -} - // Allow specifying the external location for the native libraries // The libraries should be properly sorted into subfolders corresponding to the `libraryTargetPath` value for each // os/arch/libc combination @@ -104,23 +98,47 @@ tasks.register('copyExternalLibs', Copy) { tasks.register('assembleAll') {} -// use the build config names to create configurations, copy lib and asemble jar tasks +// gtest is 'specific' - we need to prepare it for each build configuration becuase it must be compiled with the same flags +buildConfigs().each { cfg -> + def name = cfg.name + def compilerFlags = cfg.compilerArgs.findAll { + return it.contains('sanitize') + }?.join(' ') + + def prepareGTestTask = tasks.register("prepareGTest${capitalizeFirstLetter(name)}", Exec) { + environment 'CMAKE_CXX_STANDARD', '11' + if (osIdentifier() == 'macos') { + environment "CMAKE_OSX_DEPLOYMENT_TARGET", "10.10" + } + if (!compilerFlags.isEmpty()) { + environment "CMAKE_CXX_FLAGS", compilerFlags + environment "CMAKE_C_FLAGS", compilerFlags + environment "CMAKE_EXE_LINKER_FLAGS", compilerFlags + environment "CMAKE_SHARED_LINKER_FLAGS", compilerFlags + } -buildConfigNames().each { name -> - configurations.create(name) { - canBeConsumed = true - canBeResolved = false - extendsFrom configurations.implementation + dependsOn unzipGTest + workingDir gtestDir + commandLine "${projectDir}/src/test/prepare_gtest.sh", name, gtestDir } - def gtestTask = tasks.register("gtest${capitalizeFirstLetter(name)}", Exec) { onlyIf { !project.hasProperty('skip-tests') && !project.hasProperty('skip-cpp-tests') } - dependsOn prepareGTest + dependsOn prepareGTestTask workingDir projectDir - commandLine "./src/test/run_tests.sh", projectDir, name, gtestDir + commandLine "${projectDir}/src/test/run_tests.sh", name, projectDir, gtestDir, compilerFlags } +} + +// use the build config names to create configurations, copy lib and asemble jar tasks + +buildConfigNames().each { name -> + configurations.create(name) { + canBeConsumed = true + canBeResolved = false + extendsFrom configurations.implementation } + def copyTask = tasks.register("copy${capitalizeFirstLetter(name)}Libs", Copy) { from file(librarySourcePath(name, name == 'release' ? 'stripped' : '')).parent // the release build is stripped into file(libraryTargetPath(name)) @@ -130,7 +148,7 @@ buildConfigNames().each { name -> description = "Assemble the ${name} build of the library" dependsOn copyExternalLibs if (!project.hasProperty('skip-native')) { - dependsOn copyTask // tasks.named("copy${capitalizeFirstLetter(name)}Libs") + dependsOn copyTask } from sourceSets.main.output.classesDirs from files(libraryTargetBase(name)) { diff --git a/ddprof-lib/src/test/prepare_gtest.sh b/ddprof-lib/src/test/prepare_gtest.sh new file mode 100755 index 000000000..b0d508455 --- /dev/null +++ b/ddprof-lib/src/test/prepare_gtest.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# set -x +CONFIG_NAME=$1 +GTEST_DIR=$2 + +CFG_DIR="${GTEST_DIR}"/configs/"${CONFIG_NAME}" +if [ ! -d "${CFG_DIR}/lib" ]; then + mkdir -p $CFG_DIR + cd "${GTEST_DIR}"/configs/"${CONFIG_NAME}" + cmake ../.. + make +fi diff --git a/ddprof-lib/src/test/run_tests.sh b/ddprof-lib/src/test/run_tests.sh index 739b5ebb7..d4b62ff91 100755 --- a/ddprof-lib/src/test/run_tests.sh +++ b/ddprof-lib/src/test/run_tests.sh @@ -1,8 +1,12 @@ #!/usr/bin/env bash -PROJECT_DIR=$1 -CONFIG_NAME=$2 +CONFIG_NAME=$1 +PROJECT_DIR=$2 GTEST_DIR=$3 +shift +shift +shift +ARGS=$@ TMPDIR="${TMPDIR:-$(dirname $(mktemp))}" @@ -10,19 +14,20 @@ function build_test() { NAME=$1 echo "Building test ${NAME}..." # shellcheck disable=SC2038 - find ${PROJECT_DIR}/build/obj/main/${CONFIG_NAME} -name "*.o" | - xargs g++ -std=c++11 -o ${TMPDIR}/${NAME} ${PROJECT_DIR}/src/test/cpp/${NAME}.cpp \ - -I${GTEST_DIR}/googletest/include -I${GTEST_DIR}/googlemock/include \ - -I${PROJECT_DIR}/src/main/cpp ${GTEST_DIR}/lib/libgtest.a \ - ${GTEST_DIR}/lib/libgtest_main.a ${GTEST_DIR}/lib/libgmock.a \ - ${GTEST_DIR}/lib/libgmock_main.a -pthread + files=$(find ${PROJECT_DIR}/build/obj/main/${CONFIG_NAME} -name "*.o" -print | tr '\n' ' ') + libdir=${GTEST_DIR}/configs/${CONFIG_NAME}/lib + g++ -std=c++11 ${ARGS[@]} -o ${TMPDIR}/${NAME}_${CONFIG_NAME} ${PROJECT_DIR}/src/test/cpp/${NAME}.cpp \ + -I${GTEST_DIR}/googletest/include -I${GTEST_DIR}/googlemock/include \ + -I${PROJECT_DIR}/src/main/cpp ${libdir}/libgtest.a \ + ${libdir}/libgtest_main.a ${libdir}/libgmock.a \ + ${libdir}/libgmock_main.a $files -lpthread -lm -lz -lrt -Wl,--no-as-needed -ldl -pthread } function build_and_run() { NAME=$1 build_test ${NAME} echo "Running test ${NAME}..." - ${TMPDIR}/${NAME} + ${TMPDIR}/${NAME}_${CONFIG_NAME} } build_and_run ddprof_ut diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index 81d84618a..3dd600756 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -59,6 +59,7 @@ buildConfigurations.each { config -> // Exclude patterns for ASAN and TSAN tests if (name.contains('asan')) { exclude '**/endpoints*' + exclude '**/alloc*' } if (name.contains('tsan')) { exclude '**/cpu*' diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index e4873b695..d905e1b8e 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -15,6 +15,12 @@ def buildConfigNames() { }.collect { it.name }.toSet() } +def buildConfigs() { + return buildConfigurations.findAll { + it.os == osIdentifier() && it.arch == archIdentifier() + } +} + def locateLibrary(String libName) { if (os().isLinux()) { try { @@ -60,6 +66,7 @@ ext { hasAsan = this.&hasAsan capitalizeFirstLetter = this.&capitalizeFirstLetter buildConfigNames = this.&buildConfigNames + buildConfigs = this.&buildConfigs } // ======= Define build configurations below ======== From 611829275780a8324ef1e4b592a217e115953683 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 27 May 2024 18:08:29 +0000 Subject: [PATCH 30/80] Remove -lz --- ddprof-lib/src/test/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddprof-lib/src/test/run_tests.sh b/ddprof-lib/src/test/run_tests.sh index d4b62ff91..7de8c6bf7 100755 --- a/ddprof-lib/src/test/run_tests.sh +++ b/ddprof-lib/src/test/run_tests.sh @@ -20,7 +20,7 @@ function build_test() { -I${GTEST_DIR}/googletest/include -I${GTEST_DIR}/googlemock/include \ -I${PROJECT_DIR}/src/main/cpp ${libdir}/libgtest.a \ ${libdir}/libgtest_main.a ${libdir}/libgmock.a \ - ${libdir}/libgmock_main.a $files -lpthread -lm -lz -lrt -Wl,--no-as-needed -ldl -pthread + ${libdir}/libgmock_main.a $files -lpthread -lm -lrt -Wl,--no-as-needed -ldl -pthread } function build_and_run() { From 90563855fb21b4cfdd5a3ca92b9423ec879dfbdf Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 27 May 2024 20:24:04 +0200 Subject: [PATCH 31/80] More gtest build compatibilty fixes --- ddprof-lib/build.gradle | 13 +++++++++++-- ddprof-lib/src/test/run_tests.sh | 4 ++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index a9ba37920..1abf1f762 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -53,6 +53,12 @@ ext { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 +java { + toolchain { + languageVersion = JavaLanguageVersion.of(11) + } +} + def isGitlabCI = System.getenv("GITLAB_CI") != null def buildTempDir = "${projectDir}/build/tmp" def gtestVersion = "release-1.12.1" @@ -127,7 +133,7 @@ buildConfigs().each { cfg -> } dependsOn prepareGTestTask workingDir projectDir - commandLine "${projectDir}/src/test/run_tests.sh", name, projectDir, gtestDir, compilerFlags + commandLine "${projectDir}/src/test/run_tests.sh", name, projectDir, gtestDir, compilerFlags, osIdentifier() == 'linux' ? '-lrt -Wl,--no-as-needed' : '' } } @@ -367,7 +373,10 @@ tasks.withType(Test) { if (javaHome == null) { javaHome = System.getenv("JAVA_HOME") } - executable = new File("${javaHome}", 'bin/java') + executable = file("${javaHome}/bin/java") + javaLauncher.set(javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(11) + }) } artifacts { diff --git a/ddprof-lib/src/test/run_tests.sh b/ddprof-lib/src/test/run_tests.sh index 7de8c6bf7..8b68943ea 100755 --- a/ddprof-lib/src/test/run_tests.sh +++ b/ddprof-lib/src/test/run_tests.sh @@ -16,11 +16,11 @@ function build_test() { # shellcheck disable=SC2038 files=$(find ${PROJECT_DIR}/build/obj/main/${CONFIG_NAME} -name "*.o" -print | tr '\n' ' ') libdir=${GTEST_DIR}/configs/${CONFIG_NAME}/lib - g++ -std=c++11 ${ARGS[@]} -o ${TMPDIR}/${NAME}_${CONFIG_NAME} ${PROJECT_DIR}/src/test/cpp/${NAME}.cpp \ + g++ -std=c++11 -o ${TMPDIR}/${NAME}_${CONFIG_NAME} ${PROJECT_DIR}/src/test/cpp/${NAME}.cpp \ -I${GTEST_DIR}/googletest/include -I${GTEST_DIR}/googlemock/include \ -I${PROJECT_DIR}/src/main/cpp ${libdir}/libgtest.a \ ${libdir}/libgtest_main.a ${libdir}/libgmock.a \ - ${libdir}/libgmock_main.a $files -lpthread -lm -lrt -Wl,--no-as-needed -ldl -pthread + ${libdir}/libgmock_main.a $files -lpthread -lm -ldl ${ARGS[@]} -pthread } function build_and_run() { From f9b3e7b2909ad5db6ab8f2f5a1e39b702edf12cd Mon Sep 17 00:00:00 2001 From: r1viollet Date: Tue, 28 May 2024 10:06:41 +0200 Subject: [PATCH 32/80] Minor fix on folder used to run the gtest build --- ddprof-lib/src/test/CMakeLists.txt | 3 --- ddprof-lib/src/test/prepare_gtest.sh | 29 +++++++++++++++++++++------- gradle/configurations.gradle | 4 ++-- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/ddprof-lib/src/test/CMakeLists.txt b/ddprof-lib/src/test/CMakeLists.txt index 5c9e401d6..41fb5afc2 100644 --- a/ddprof-lib/src/test/CMakeLists.txt +++ b/ddprof-lib/src/test/CMakeLists.txt @@ -13,9 +13,6 @@ include(./AddUnitTests.cmake) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_AUTOMOC ON) -set(CMAKE_AUTORCC ON) - include(FetchContent) FetchContent_Declare( googletest diff --git a/ddprof-lib/src/test/prepare_gtest.sh b/ddprof-lib/src/test/prepare_gtest.sh index b0d508455..de98b2e8c 100755 --- a/ddprof-lib/src/test/prepare_gtest.sh +++ b/ddprof-lib/src/test/prepare_gtest.sh @@ -1,12 +1,27 @@ #!/usr/bin/env bash -# set -x +set -eo pipefail + CONFIG_NAME=$1 GTEST_DIR=$2 -CFG_DIR="${GTEST_DIR}"/configs/"${CONFIG_NAME}" -if [ ! -d "${CFG_DIR}/lib" ]; then - mkdir -p $CFG_DIR - cd "${GTEST_DIR}"/configs/"${CONFIG_NAME}" - cmake ../.. - make +CFG_DIR="${GTEST_DIR}/configs/${CONFIG_NAME}" + +echo "Starting script with CONFIG_NAME=${CONFIG_NAME} and GTEST_DIR=${GTEST_DIR}" + +if [ ! -d "${CFG_DIR}" ]; then + mkdir -p "${CFG_DIR}" +fi +cd "${CFG_DIR}" +echo "Running cmake in ${CFG_DIR}" +cmake ${GTEST_DIR}/../../../src/test -DCMAKE_BUILD_TYPE=${CONFIG_NAME} +if [ $? -ne 0 ]; then + echo "cmake failed" + exit 1 +fi +echo "Running make in ${CFG_DIR}" +make +if [ $? -ne 0 ]; then + echo "make failed" + exit 1 fi +echo "Build completed successfully." diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index dd9ef2596..96e6d135e 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -53,10 +53,10 @@ def libasan = locateLibasan() def libtsan = locateLibtsan() if (libasan == null) { - logger.lifecycle("Asan library not found") + logger.lifecycle("Asan library not found") } if (libtsan == null) { - logger.lifecycle("Tsan library not found") + logger.lifecycle("Tsan library not found") } def hasAsan() { return libasan != null From 8f5769ab406d651cbdd2ec47a68422ac51d7a0e3 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Tue, 28 May 2024 10:28:10 +0200 Subject: [PATCH 33/80] Remove minor leak in unit test --- ddprof-lib/src/test/cpp/ddprof_ut.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/ddprof-lib/src/test/cpp/ddprof_ut.cpp b/ddprof-lib/src/test/cpp/ddprof_ut.cpp index 15eae9672..e3bd0cd54 100644 --- a/ddprof-lib/src/test/cpp/ddprof_ut.cpp +++ b/ddprof-lib/src/test/cpp/ddprof_ut.cpp @@ -79,6 +79,7 @@ // long string should have been truncated to 8191 characters int prefix = 1 + (31 - __builtin_clz(8191)) / 7 + 1; EXPECT_EQ(0, buf[0].data()[prefix + 8191]); + free(str); } TEST(OS, threadId_sanity) { From ea612632f62be4902bd178312d1b505fd7bc1da2 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 10:31:14 +0200 Subject: [PATCH 34/80] Fix CI build typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa797be0a..541af6633 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -253,7 +253,7 @@ jobs: export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=musl/${{ matrix.java_version }} export LIBC=musl - export JAVA_TEST_HOME=$(pwd)/test_jdkTry + export JAVA_TEST_HOME=$(pwd)/test_jdk ./gradlew :ddprof-test:test --info - name: Upload logs uses: actions/upload-artifact@v3 From 5a19586d995b2af3f6dc614f26dab5e9783aa613 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 10:36:47 +0200 Subject: [PATCH 35/80] Allow excluding tests for sanitized configs --- .../java/com/datadoghq/profiler/AbstractProfilerTest.java | 8 ++++++++ .../datadoghq/profiler/alloc/AllocationProfilerTest.java | 1 + 2 files changed, 9 insertions(+) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java index bbe31e226..3d4fb6142 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java @@ -129,6 +129,14 @@ private static Duration parseInterval(String command, String part) { return Duration.ofMillis(0); } + protected final boolean isAsan() { + return System.getenv("ASAN_OPTIONS") != null; + } + + protected final boolean isTsan() { + return System.getenv("TSAN_OPTIONS") != null; + } + @BeforeEach public void setupProfiler() throws Exception { jfrDump = Files.createTempFile(Paths.get("/tmp"), getClass().getName() + UUID.randomUUID(), ".jfr"); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/alloc/AllocationProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/alloc/AllocationProfilerTest.java index e260eb0b6..79901e22b 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/alloc/AllocationProfilerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/alloc/AllocationProfilerTest.java @@ -19,6 +19,7 @@ public class AllocationProfilerTest extends AbstractProfilerTest { public void shouldGetObjectAllocationSamples() throws InterruptedException { Assumptions.assumeFalse(Platform.isJ9() || Platform.isZing()); Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); + Assumptions.assumeFalse(isAsan() || isTsan()); AllocatingTarget target1 = new AllocatingTarget(); AllocatingTarget target2 = new AllocatingTarget(); From bc6648457279c128eabe8d17361dd6df30298ce7 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 11:15:49 +0200 Subject: [PATCH 36/80] Add comment about Buffer var-size-array --- ddprof-lib/src/main/cpp/buffers.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ddprof-lib/src/main/cpp/buffers.h b/ddprof-lib/src/main/cpp/buffers.h index f45ae5a55..c656bf4c7 100644 --- a/ddprof-lib/src/main/cpp/buffers.h +++ b/ddprof-lib/src/main/cpp/buffers.h @@ -29,6 +29,8 @@ class Buffer { private: int _offset; static const int _limit = BUFFER_SIZE - sizeof(int); + // this array is 'extended' by the RecordingBuffer + // this will confuse sanitizers and most of the sane people but it seems to work char _data[_limit]; public: From 756b90307d51ab0c95a9d1405f67cab9e29e095c Mon Sep 17 00:00:00 2001 From: r1viollet Date: Tue, 28 May 2024 11:52:21 +0200 Subject: [PATCH 37/80] Asan minor fixes - Ensure we use memcpy dut to alignment issues - Silence the access in VM structs until we solve it as a separate ticket --- ddprof-lib/src/main/cpp/stackFrame_x64.cpp | 4 +++- ddprof-lib/src/main/cpp/vmStructs.h | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/stackFrame_x64.cpp b/ddprof-lib/src/main/cpp/stackFrame_x64.cpp index c059838ce..e48c41818 100644 --- a/ddprof-lib/src/main/cpp/stackFrame_x64.cpp +++ b/ddprof-lib/src/main/cpp/stackFrame_x64.cpp @@ -224,7 +224,9 @@ bool StackFrame::checkInterruptedSyscall() { // mov eax, SYS_ppoll with any timeout (ppoll adjusts timeout automatically) uintptr_t pc = this->pc(); if ((pc & 0xfff) >= 7 && *(instruction_t*)(pc - 7) == 0xb8) { - int nr = *(int*)(pc - 6); + // mainly to satisfy sanitizer complaints around alignment issue: we rely on memcpy + int nr = 0; + memcpy(&nr, (void*)(pc - 6), sizeof(nr)); if (nr == SYS_ppoll || (nr == SYS_poll && (int)REG(RDX, rdx) == -1) || (nr == SYS_epoll_wait && (int)REG(R10, r10) == -1) diff --git a/ddprof-lib/src/main/cpp/vmStructs.h b/ddprof-lib/src/main/cpp/vmStructs.h index c6b17e0a1..3eb3fc8e6 100644 --- a/ddprof-lib/src/main/cpp/vmStructs.h +++ b/ddprof-lib/src/main/cpp/vmStructs.h @@ -444,6 +444,9 @@ class VMThread : VMStructs { return *(int*)(osthread + _osthread_id_offset); } + // This triggers in the test suite due to the absence of robust thread filters. + // We will address this as a separate ticket. + __attribute__((no_sanitize("address"))) int state() { return _thread_state_offset >= 0 ? *(int*) at(_thread_state_offset) : 0; } From a786605aa7b653c08a6abb0240efee91989b233e Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 12:37:05 +0200 Subject: [PATCH 38/80] Use the cmake defs from the gtest suite --- ddprof-lib/build.gradle | 5 +++-- ddprof-lib/src/test/prepare_gtest.sh | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 1abf1f762..cef742c9d 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -112,9 +112,10 @@ buildConfigs().each { cfg -> }?.join(' ') def prepareGTestTask = tasks.register("prepareGTest${capitalizeFirstLetter(name)}", Exec) { + def args = [] environment 'CMAKE_CXX_STANDARD', '11' if (osIdentifier() == 'macos') { - environment "CMAKE_OSX_DEPLOYMENT_TARGET", "10.10" + args += "-DCMAKE_OSX_DEPLOYMENT_TARGET=10.10" } if (!compilerFlags.isEmpty()) { environment "CMAKE_CXX_FLAGS", compilerFlags @@ -125,7 +126,7 @@ buildConfigs().each { cfg -> dependsOn unzipGTest workingDir gtestDir - commandLine "${projectDir}/src/test/prepare_gtest.sh", name, gtestDir + commandLine "${projectDir}/src/test/prepare_gtest.sh", name, gtestDir, args.join(' ') } def gtestTask = tasks.register("gtest${capitalizeFirstLetter(name)}", Exec) { onlyIf { diff --git a/ddprof-lib/src/test/prepare_gtest.sh b/ddprof-lib/src/test/prepare_gtest.sh index de98b2e8c..be6c8ed6c 100755 --- a/ddprof-lib/src/test/prepare_gtest.sh +++ b/ddprof-lib/src/test/prepare_gtest.sh @@ -3,6 +3,8 @@ set -eo pipefail CONFIG_NAME=$1 GTEST_DIR=$2 +shift +ARGS=$@ CFG_DIR="${GTEST_DIR}/configs/${CONFIG_NAME}" @@ -13,7 +15,7 @@ if [ ! -d "${CFG_DIR}" ]; then fi cd "${CFG_DIR}" echo "Running cmake in ${CFG_DIR}" -cmake ${GTEST_DIR}/../../../src/test -DCMAKE_BUILD_TYPE=${CONFIG_NAME} +cmake ../.. ${ARGS[@]} -DCMAKE_BUILD_TYPE=${CONFIG_NAME} if [ $? -ne 0 ]; then echo "cmake failed" exit 1 From 06051967a0d9a828f6e0d3f825390bc1183f59c8 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 12:37:36 +0200 Subject: [PATCH 39/80] Do not delegate ddprof-test:test to ddprof-test:testDebug --- .github/workflows/ci.yml | 12 ++++++------ ddprof-test/build.gradle | 9 ++++----- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 541af6633..ce78aa2c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,7 @@ jobs: export LIBC=glibc export JAVA_TEST_HOME=$(pwd)/test_jdk export BUILD_TYPE=SanitizedDebug - ./gradlew :ddprof-test:test --info + ./gradlew :ddprof-test:testDebug --info - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -110,7 +110,7 @@ jobs: export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH export BUILD_TYPE=SanitizedDebug - ./gradlew :ddprof-test:test --info + ./gradlew :ddprof-test:testDebug --info - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -156,7 +156,7 @@ jobs: export LIBC=glibc export BUILD_TYPE=SanitizedDebug chmod a+x gradlew - ./gradlew :ddprof-test:test --info + ./gradlew :ddprof-test:testDebug --info - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -203,7 +203,7 @@ jobs: export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH export BUILD_TYPE=SanitizedDebug - ./gradlew :ddprof-test:test --info + ./gradlew :ddprof-test:testDebug --info - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -254,7 +254,7 @@ jobs: export TEST_CONFIGURATION=musl/${{ matrix.java_version }} export LIBC=musl export JAVA_TEST_HOME=$(pwd)/test_jdk - ./gradlew :ddprof-test:test --info + ./gradlew :ddprof-test:testDebug --info - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -317,7 +317,7 @@ jobs: export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH export BUILD_TYPE=SanitizedDebug - ./gradlew :ddprof-test:test --info + ./gradlew :ddprof-test:testDebug --info - name: Upload logs uses: actions/upload-artifact@v3 if: always() diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index 048491e75..12b58b092 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -55,10 +55,9 @@ buildConfigurations.each { config -> } logger.lifecycle("Setting environment variables for ${name}: ${config.testEnv}") } - } - if (name == 'debug') { - // redirect the default test task to the debug test task - tasks.test.dependsOn task.get() + systemProperties = tasks.test.systemProperties + jvmArgs = tasks.test.jvmArgs + environment = tasks.test.environment } } @@ -86,7 +85,7 @@ tasks.withType(Test).configureEach { test { onlyIf { - false // disable the default test task + false } } From aa1d0f65f77d218eb2bbdcef99a083b4bf380133 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 13:09:06 +0200 Subject: [PATCH 40/80] Allow gradle up-to-date checks for custom tasks --- ddprof-lib/build.gradle | 16 +++++++++++++++- ddprof-test/build.gradle | 4 ++-- gradle/configurations.gradle | 4 ++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index cef742c9d..12f4e2d22 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -67,7 +67,7 @@ def gtestLib = "${gtestDir}/build/lib/libgtest.a" tasks.register('downloadGTest', Download) { src "https://github.com/google/googletest/archive/refs/tags/${gtestVersion}.zip" - dest "${projectDir}/build/tmp/gtest.zip" + dest "${buildTempDir}/gtest.zip" overwrite false } @@ -127,6 +127,8 @@ buildConfigs().each { cfg -> dependsOn unzipGTest workingDir gtestDir commandLine "${projectDir}/src/test/prepare_gtest.sh", name, gtestDir, args.join(' ') + inputs.file file("${buildTempDir}/gtest.zip") + outputs.dir file("${gtestDir}/configs/${name}") } def gtestTask = tasks.register("gtest${capitalizeFirstLetter(name)}", Exec) { onlyIf { @@ -135,6 +137,7 @@ buildConfigs().each { cfg -> dependsOn prepareGTestTask workingDir projectDir commandLine "${projectDir}/src/test/run_tests.sh", name, projectDir, gtestDir, compilerFlags, osIdentifier() == 'linux' ? '-lrt -Wl,--no-as-needed' : '' + outputs.upToDateWhen {true} } } @@ -193,11 +196,18 @@ tasks.whenTaskAdded { task -> includes = task.includes systemIncludes = task.systemIncludes source = task.source + inputs.files source + outputs.dir objectFileDir } def linkTask = tasks.findByName("linkLib${capitalizeFirstLetter(config.name)}".toString()) if (linkTask != null) { linkTask.dependsOn thisTask } + def gtestTask = tasks.findByName("gtest${capitalizeFirstLetter(config.name)}".toString()) + if (gtestTask != null) { + gtestTask.dependsOn thisTask.get() + gtestTask.inputs.dir thisTask.get().objectFileDir + } } } } @@ -218,6 +228,8 @@ tasks.whenTaskAdded { task -> toolChain = task.toolChain targetPlatform = task.targetPlatform libs = task.libs + inputs.files source + outputs.file linkedFile } if (config.name == 'release') { tasks.register('stripLibRelease', StripSymbols) { @@ -226,6 +238,8 @@ tasks.whenTaskAdded { task -> toolChain = tasks.linkLibRelease.toolChain binaryFile = tasks.linkLibRelease.linkedFile.get() outputFile = file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/stripped/libjavaProfiler.${os().isLinux() ? 'so' : 'dylib'}") + inputs.file binaryFile + outputs.file outputFile } } } diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index 12b58b092..fe7010117 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -30,7 +30,7 @@ buildConfigurations.each { config -> if (config.os != osIdentifier() || config.arch != archIdentifier()) { return } - logger.lifecycle("Creating configuration for ${name}") + logger.debug("Creating configuration for ${name}") def cfg = configurations.create("test${capitalizeFirstLetter(name)}Implementation") { canBeConsumed = true @@ -53,7 +53,7 @@ buildConfigurations.each { config -> config.testEnv.each { key, value -> environment key, value } - logger.lifecycle("Setting environment variables for ${name}: ${config.testEnv}") + logger.debug("Setting environment variables for ${name}: ${config.testEnv}") } systemProperties = tasks.test.systemProperties jvmArgs = tasks.test.jvmArgs diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index 96e6d135e..576131bed 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -53,10 +53,10 @@ def libasan = locateLibasan() def libtsan = locateLibtsan() if (libasan == null) { - logger.lifecycle("Asan library not found") + logger.debug("Asan library not found") } if (libtsan == null) { - logger.lifecycle("Tsan library not found") + logger.debug("Tsan library not found") } def hasAsan() { return libasan != null From 0f140e84a9135c54fd441d25dfb020f0fa282643 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 13:09:16 +0200 Subject: [PATCH 41/80] Remove cmake files --- ddprof-lib/src/test/AddUnitTests.cmake | 84 -------------------------- ddprof-lib/src/test/CMakeLists.txt | 62 ------------------- ddprof-lib/src/test/Sanitizers.cmake | 62 ------------------- 3 files changed, 208 deletions(-) delete mode 100644 ddprof-lib/src/test/AddUnitTests.cmake delete mode 100644 ddprof-lib/src/test/CMakeLists.txt delete mode 100644 ddprof-lib/src/test/Sanitizers.cmake diff --git a/ddprof-lib/src/test/AddUnitTests.cmake b/ddprof-lib/src/test/AddUnitTests.cmake deleted file mode 100644 index 75977a1ed..000000000 --- a/ddprof-lib/src/test/AddUnitTests.cmake +++ /dev/null @@ -1,84 +0,0 @@ -#[[ Create an executable -Syntax: -add_exe( src1 [src2 ...] [LIBRARIES lib1 lib2 ...] [DEFINITIONS def1 def2]) -will compile an executable named from source files src1 src2... -with pre-processor definitions def1 def2 (-Ddef1 -Ddef2 ... will be added to compile command) -and link against lib1 lib2 ...and libm - -Examples: -add_exe(myexe src1.cpp) -add_exe(myexe src1.cpp - LIBRARIES ${CMAKE_SOURCE_DIR}/myLib - DEFINITIONS UNIT_TEST) -#]] -function(add_exe name) - set(cur_var "sources") - set(exe_sources "") - set(exe_libraries "") - set(exe_definitions "") - set(exe_include_dirs "") - - foreach(arg IN LISTS ARGN) - if(arg STREQUAL "LIBRARIES") - set(cur_var "libraries") - elseif(arg STREQUAL "DEFINITIONS") - set(cur_var "definitions") - else() - list(APPEND exe_${cur_var} ${arg}) - - if(cur_var STREQUAL "sources") - get_filename_component(src_dir ${arg} DIRECTORY) - list(APPEND exe_include_dirs ${src_dir}) - endif() - endif() - endforeach() - - add_executable(${name} ${exe_sources}) - set_target_properties(${name} PROPERTIES COMPILE_DEFINITIONS "${exe_definitions}") - target_link_libraries(${name} PRIVATE ${exe_libraries}) - list(REMOVE_DUPLICATES exe_include_dirs) - target_include_directories(${name} PRIVATE ${exe_include_dirs}) -endfunction() - -# Define functions - -#[[ Create a unit test -Syntax: -add_unit_test( src1 [src2 ...] [LIBRARIES lib1 lib2 ...] [DEFINITIONS def1 def2]) -will compile an unit test named from source files src1 src2... -with pre-processor definitions def1 def2 (-Ddef1 -Ddef2 ... will be added to compile command) -and link against lib1 lib2 ... and libm - -Examples: -add_unit_test(myexe src1.cpp) -add_unit_test(myexe src1.cpp DEFINITIONS UNIT_TEST) -#]] -function(add_unit_test name) - set(options NO_DETECT_LEAKS) - set(oneValueArgs) - set(multiValueArgs) - cmake_parse_arguments(PARSE_ARGV 1 MY "${options}" "${oneValueArgs}" "${multiValueArgs}") - message(STATUS "Creating unit test : " ${name}) - - ## Create exe with sources. - add_exe(${name} ${MY_UNPARSED_ARGUMENTS}) - - target_link_libraries(${name} PRIVATE gtest Threads::Threads gmock_main gmock) - target_include_directories(${name} PRIVATE ../main/cpp - ${GTEST_INCLUDE_DIRS}) - - add_test(NAME ${name} COMMAND ${name}) - set_tests_properties( - ${name} - PROPERTIES - ENVIRONMENT - "UBSAN_OPTIONS=halt_on_error=1 abort_on_error=1 print_stacktrace=1;LSAN_OPTIONS=detect_leaks=$ malloc_context_size=2 print_suppressions=0;ASAN_OPTIONS=suppressions=${CMAKE_SOURCE_DIR}/cmake/asan.supp" - ) -endfunction() - -function(add_java_unit_test name) - add_unit_test(${name} ${ARGN}) - target_include_directories(${name} PRIVATE - $ENV{JAVA_HOME}/include - $ENV{JAVA_HOME}/include/${OS_SUFFIX}) -endfunction() diff --git a/ddprof-lib/src/test/CMakeLists.txt b/ddprof-lib/src/test/CMakeLists.txt deleted file mode 100644 index 41fb5afc2..000000000 --- a/ddprof-lib/src/test/CMakeLists.txt +++ /dev/null @@ -1,62 +0,0 @@ -cmake_minimum_required(VERSION 3.14) -project(ddprof-lib-test) - -# GoogleTest requires at least C++14 -set(CMAKE_CXX_STANDARD 14) -set(CMAKE_C_STANDARD 11) - -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(Threads REQUIRED) - -include(./Sanitizers.cmake) -include(./AddUnitTests.cmake) - -set(CMAKE_CXX_STANDARD_REQUIRED ON) - -include(FetchContent) -FetchContent_Declare( - googletest - URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip -) -# For Windows: Prevent overriding the parent project's compiler/linker settings -set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) -FetchContent_MakeAvailable(googletest) - -enable_testing() - -set(OS_SUFFIX "linux") -if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(OS_SUFFIX "macos") -endif() - -# uncomment the following lines to print out various build related variables -# include(CMakePrintHelpers) -# cmake_print_variables(SRC_FILES PROJECT_SOURCE_DIR CMAKE_SYSTEM_NAME) - -# Include all *.cpp files from the cpp folder -file(GLOB_RECURSE TEST_FILES CONFIGURE_DEPENDS - "${PROJECT_SOURCE_DIR}/cpp/*.cpp" -) - -# not all source files are required to run gtests - -# new source files should be added when a test is complaining about missing symbols -file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS - "${PROJECT_SOURCE_DIR}/../main/cpp/os_${OS_SUFFIX}.cpp" - "${PROJECT_SOURCE_DIR}/../main/cpp/context.cpp" - "${PROJECT_SOURCE_DIR}/../main/cpp/rustDemangler.cpp" - "${PROJECT_SOURCE_DIR}/../main/cpp/counters.cpp" - "${PROJECT_SOURCE_DIR}/../main/cpp/threadFilter.cpp" - "${PROJECT_SOURCE_DIR}/../main/cpp/dictionary.cpp" - "${PROJECT_SOURCE_DIR}/../main/cpp/methodCache.cpp" - "${PROJECT_SOURCE_DIR}/../main/cpp/mutex.cpp" - "${PROJECT_SOURCE_DIR}/../main/cpp/threadInfo.cpp" -) - -add_compile_definitions(DEBUG) -add_compile_definitions(COUNTERS) - -## Declare unit tests with add_unit_test / add_java_unit_test -add_unit_test(ddprof-lib-test - NO_DETECT_LEAKS - ${TEST_FILES} - ${SRC_FILES}) diff --git a/ddprof-lib/src/test/Sanitizers.cmake b/ddprof-lib/src/test/Sanitizers.cmake deleted file mode 100644 index 5938fb60b..000000000 --- a/ddprof-lib/src/test/Sanitizers.cmake +++ /dev/null @@ -1,62 +0,0 @@ -# Unless explicitly stated otherwise all files in this repository are licensed under the Apache -# License Version 2.0. This product includes software developed at Datadog -# (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. - -set(SAN_FLAGS "-fsanitize=undefined -fsanitize=float-divide-by-zero -fno-sanitize-recover") -set(ASAN_FLAGS "-fsanitize=address") -set(TSAN_FLAGS "-fsanitize=thread") -set(STACK_FLAGS "-fstack-protector-all") - -list(APPEND CMAKE_CONFIGURATION_TYPES SanitizedDebug ThreadSanitizedDebug) -message(STATUS "Adding SanitizedDebug, ThreadSanitizedDebug as build types") - -# Add flags for sanitized debug (asan) -set(CMAKE_CXX_FLAGS_SANITIZEDDEBUG - "${SAN_FLAGS} ${ASAN_FLAGS} ${STACK_FLAGS}" - CACHE STRING "Flags used by the C++ compiler during sanitized builds." FORCE) -set(CMAKE_C_FLAGS_SANITIZEDDEBUG - "${SAN_FLAGS} ${ASAN_FLAGS} ${STACK_FLAGS}" - CACHE STRING "Flags used by the C compiler during sanitized builds." FORCE) -set(CMAKE_EXE_LINKER_FLAGS_SANITIZEDDEBUG - "" - CACHE STRING "Flags used for linking binaries during sanitized builds." FORCE) -set(CMAKE_SHARED_LINKER_FLAGS_SANITIZEDDEBUG - "" - CACHE STRING "Flags used by the shared libraries linker during sanitized builds." FORCE) - -# Add flags for thread-sanized debug -set(CMAKE_CXX_FLAGS_THREADSANITIZEDDEBUG - "${SAN_FLAGS} ${TSAN_FLAGS} ${STACK_FLAGS}" - CACHE STRING "Flags used by the C++ compiler during sanitized builds." FORCE) -set(CMAKE_C_FLAGS_THREADSANITIZEDDEBUG - "${SAN_FLAGS} ${TSAN_FLAGS} ${STACK_FLAGS}" - CACHE STRING "Flags used by the C compiler during sanitized builds." FORCE) -set(CMAKE_EXE_LINKER_FLAGS_THREADSANITIZEDDEBUG - "" - CACHE STRING "Flags used for linking binaries during sanitized builds." FORCE) -set(CMAKE_SHARED_LINKER_FLAGS_THREADSANITIZEDDEBUG - "" - CACHE STRING "Flags used by the shared libraries linker during sanitized builds." FORCE) - -string(REPLACE ";" " " LD_FLAGS_STR "${LD_FLAGS}") - -if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_EXE_LINKER_FLAGS_SANITIZEDDEBUG - "${CMAKE_EXE_LINKER_FLAGS_SANITIZEDDEBUG} -static-libasan -static-libubsan -static-liblsan") - set(CMAKE_SHARED_LINKER_FLAGS_SANITIZEDDEBUG - "${CMAKE_SHARED_LINKER_FLAGS_SANITIZEDDEBUG} -static-libasan -static-libubsan -static-liblsan") - set(CMAKE_EXE_LINKER_FLAGS_THREADSANITIZEDDEBUG - "${CMAKE_EXE_LINKER_FLAGS_THREADSANITIZEDDEBUG} -static-libtsan -static-libubsan") - set(CMAKE_SHARED_LINKER_FLAGS_THREADSANITIZEDDEBUG - "${CMAKE_SHARED_LINKER_FLAGS_THREADSANITIZEDDEBUG} -static-libtsan -static-libubsan") -endif() - -mark_as_advanced( - CMAKE_CXX_FLAGS_SANITIZEDDEBUG - CMAKE_C_FLAGS_SANITIZEDDEBUG - CMAKE_EXE_LINKER_FLAGS_SANITIZEDDEBUG - CMAKE_SHARED_LINKER_FLAGS_SANITIZEDDEBUG - CMAKE_CXX_FLAGS_THREADSANITIZEDDEBUG - CMAKE_C_FLAGS_THREADSANITIZEDDEBUG - CMAKE_EXE_LINKER_FLAGS_THREADSANITIZEDDEBUG - CMAKE_SHARED_LINKER_FLAGS_THREADSANITIZEDDEBUG) From b5af49dc506b3880e946bd9240c7671761bc2046 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 13:16:12 +0200 Subject: [PATCH 42/80] Use common sanitizer suppression files --- ddprof-lib/build.gradle | 3 +++ gradle/configurations.gradle | 6 +++--- {ddprof-test => gradle/sanitizers}/asan.supp | 0 {ddprof-test => gradle/sanitizers}/tsan.supp | 0 {ddprof-test => gradle/sanitizers}/ubsan.supp | 0 5 files changed, 6 insertions(+), 3 deletions(-) rename {ddprof-test => gradle/sanitizers}/asan.supp (100%) rename {ddprof-test => gradle/sanitizers}/tsan.supp (100%) rename {ddprof-test => gradle/sanitizers}/ubsan.supp (100%) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 12f4e2d22..e378713f6 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -134,6 +134,9 @@ buildConfigs().each { cfg -> onlyIf { !project.hasProperty('skip-tests') && !project.hasProperty('skip-cpp-tests') } + cfg.testEnv.each { key, value -> + environment key, value + } dependsOn prepareGTestTask workingDir projectDir commandLine "${projectDir}/src/test/run_tests.sh", name, projectDir, gtestDir, compilerFlags, osIdentifier() == 'linux' ? '-lrt -Wl,--no-as-needed' : '' diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index 576131bed..8e3877dde 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -94,8 +94,8 @@ def commonMacosCompilerArgs = commonLinuxCompilerArgs + ["-D_XOPEN_SOURCE", "-D_ def asanEnv = libasan != null ? ['LD_PRELOAD': libasan, // warning: stack use after return can cause slowness on arm64 - "ASAN_OPTIONS" : "detect_stack_use_after_return=1 suppressions=${projectDir}/asan.supp", - "UBSAN_OPTIONS" : "halt_on_error=1 abort_on_error=1 print_stacktrace=1 suppressions=${projectDir}/ubsan.supp", + "ASAN_OPTIONS" : "detect_stack_use_after_return=1 suppressions=${rootDir}/gradle/sanitizers/asan.supp", + "UBSAN_OPTIONS" : "halt_on_error=1 abort_on_error=1 print_stacktrace=1 suppressions=${rootDir}/gradle/sanitizers/ubsan.supp", // lsan still does not run for all tests - manually trigger on some tests "LSAN_OPTIONS" : "detect_leaks=0" ] : [:] @@ -159,7 +159,7 @@ def tsanLinkerArgs = ["-fsanitize=thread", "-fno-omit-frame-pointer" // Often r def tsanEnv = libtsan != null ? ['LD_PRELOAD': libtsan, - "TSAN_OPTIONS" : "suppressions=${projectDir}/tsan.supp" + "TSAN_OPTIONS" : "suppressions=${rootDir}/gradle/sanitizers/tsan.supp" ] : [:] // Linux diff --git a/ddprof-test/asan.supp b/gradle/sanitizers/asan.supp similarity index 100% rename from ddprof-test/asan.supp rename to gradle/sanitizers/asan.supp diff --git a/ddprof-test/tsan.supp b/gradle/sanitizers/tsan.supp similarity index 100% rename from ddprof-test/tsan.supp rename to gradle/sanitizers/tsan.supp diff --git a/ddprof-test/ubsan.supp b/gradle/sanitizers/ubsan.supp similarity index 100% rename from ddprof-test/ubsan.supp rename to gradle/sanitizers/ubsan.supp From 68f8649711521a24ab4952bf5359dc3ae2a307a9 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 13:41:41 +0200 Subject: [PATCH 43/80] Fix test env setup --- ddprof-test/build.gradle | 3 --- 1 file changed, 3 deletions(-) diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index fe7010117..f21313fd1 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -55,9 +55,6 @@ buildConfigurations.each { config -> } logger.debug("Setting environment variables for ${name}: ${config.testEnv}") } - systemProperties = tasks.test.systemProperties - jvmArgs = tasks.test.jvmArgs - environment = tasks.test.environment } } From 26d56b8e41cc50b9b872d5c69f743aae1d4344db Mon Sep 17 00:00:00 2001 From: r1viollet Date: Tue, 28 May 2024 13:58:38 +0200 Subject: [PATCH 44/80] fix for com.datadoghq.profiler.wallclock.ContextWallClockTest CallTraceStorage: Adjust the return type to read 64 bits --- ddprof-lib/src/main/cpp/callTraceStorage.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ddprof-lib/src/main/cpp/callTraceStorage.cpp b/ddprof-lib/src/main/cpp/callTraceStorage.cpp index 5ecec91cf..f816e1290 100644 --- a/ddprof-lib/src/main/cpp/callTraceStorage.cpp +++ b/ddprof-lib/src/main/cpp/callTraceStorage.cpp @@ -226,9 +226,8 @@ u32 CallTraceStorage::put(int num_frames, ASGCT_CallFrame* frames, bool truncate u32 capacity = table->capacity(); u32 slot = hash & (capacity - 1); u32 step = 0; - while (true) { - int key_value = __atomic_load_n(&keys[slot], __ATOMIC_RELAXED); + u64 key_value = __atomic_load_n(&keys[slot], __ATOMIC_RELAXED); if (key_value == hash) { // Hash matches, exit the loop break; } From 715b74e3c63ad8950a23c468dd6bfaaf4e6039c3 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 17:14:55 +0200 Subject: [PATCH 45/80] Try migrating the CI Run workflow to shared tests --- .github/workflows/ci.yml | 301 +-------------------------------------- 1 file changed, 5 insertions(+), 296 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 814eec310..05d35e08d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,308 +37,17 @@ jobs: echo "skip=true" >> "$GITHUB_OUTPUT" fi fi - - test-linux-glibc: + test-matrix: needs: check-for-pr if: needs.check-for-pr.outputs.skip != 'true' - strategy: - matrix: - java_version: [8u362+9, 11.0.18+10, 17.0.6+10, 21.0.1+12] - runs-on: ubuntu-latest - timeout-minutes: 180 - steps: - - uses: actions/checkout@v3 - - name: Prepare build JDK - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: "11" - - name: Prepare JDK ${{ matrix.java_version }} - run: | - wget -nv https://download.bell-sw.com/java/${{ matrix.java_version }}/bellsoft-jdk${{ matrix.java_version }}-linux-amd64.tar.gz -O jdk.tar.gz - tar xzf *.tar.gz - find . -type d -name 'jdk*' -maxdepth 1| xargs -I {} mv {} test_jdk - - name: Test - run: | - export TEST_COMMIT=${{ github.sha }} - export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} - export LIBC=glibc - export JAVA_TEST_HOME=$(pwd)/test_jdk - export BUILD_TYPE=SanitizedDebug - ./gradlew :ddprof-test:testDebug --info - - name: Upload logs - uses: actions/upload-artifact@v3 - if: always() - with: - name: reports-linux-glibc-${{ matrix.java_version }}.zip - path: | - ddprof-test/hs_err_* - ddprof-test/build/reports/tests - ddprof-lib/src/test/build/Testing/Temporary/LastTest.log - ddprof-lib/build/tmp/compileReleaseLinuxCpp/output.txt - - uses: actions/upload-artifact@v3 - if: success() - with: - name: x64-glibc - path: build/ - - test-ubuntu-jdk: - needs: check-for-pr - if: needs.check-for-pr.outputs.skip != 'true' - strategy: - matrix: - java_version: [11, 17, 21] - runs-on: ubuntu-22.04 - timeout-minutes: 180 - steps: - - uses: actions/checkout@v3 - - name: Prepare build JDK - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: "11" - - name: Prepare JDK ${{ matrix.java_version }} - run: | - sudo apt-get update - sudo apt-get install openjdk-${{ matrix.java_version }}-jdk - uname -r - - name: Test - run: | - export TEST_COMMIT=${{ github.sha }} - export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} - export LIBC=glibc - export JAVA_TEST_HOME=/usr/lib/jvm/java-${{ matrix.java_version }}-openjdk-amd64 - export JAVA_HOME=$JAVA_HOME - export PATH=$JAVA_HOME/bin:$PATH - export BUILD_TYPE=SanitizedDebug - ./gradlew :ddprof-test:testDebug --info - - name: Upload logs - uses: actions/upload-artifact@v3 - if: always() - with: - name: reports-linux-ubuntu-jdk-${{ matrix.java_version }}.zip - path: | - ddprof-test/hs_err_* - ddprof-test/build/reports/tests - ddprof-lib/src/test/build/Testing/Temporary/LastTest.log - ddprof-lib/build/tmp/compileReleaseLinuxCpp/output.txt - - uses: actions/upload-artifact@v3 - if: success() - with: - name: x64-ubuntu-jdk - path: build/ - - test-linux-glibc-j9: - needs: check-for-pr - if: needs.check-for-pr.outputs.skip != 'true' - strategy: - matrix: - java_version: [8, 11, 17] - runs-on: ubuntu-latest - timeout-minutes: 180 - steps: - - uses: actions/checkout@v3 - - name: Prepare test JDK - uses: actions/setup-java@v3 - with: - distribution: 'adopt-openj9' - java-version: "${{ matrix.java_version }}" - - name: Store JAVA_TEST_HOME - run: JAVA_PATH=$(which java) && echo "JAVA_TEST_HOME=${JAVA_PATH/\/bin\/java/\/}" >> $GITHUB_ENV - - name: Prepare build JDK - uses: actions/setup-java@v3 - with: - distribution: 'adopt-openj9' - java-version: "11" - - name: Test - run: | - export TEST_COMMIT=${{ github.sha }} - export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} - export LIBC=glibc - export BUILD_TYPE=SanitizedDebug - chmod a+x gradlew - ./gradlew :ddprof-test:testDebug --info - - name: Upload logs - uses: actions/upload-artifact@v3 - if: always() - with: - name: reports-linux-glibc-j9-${{ matrix.java_version }}.zip - path: | - ddprof-test/javacore* - ddprof-test/build/reports/tests - ddprof-lib/src/test/build/Testing/Temporary/LastTest.log - ddprof-lib/build/tmp/compileReleaseLinuxCpp/output.txt - - uses: actions/upload-artifact@v3 - if: success() - with: - name: x64-glibc-j9 - path: build/ - - test-linux-glibc-oracle8: - needs: check-for-pr - if: needs.check-for-pr.outputs.skip != 'true' - runs-on: ubuntu-22.04 - timeout-minutes: 180 - steps: - - uses: actions/checkout@v3 - - name: Prepare build JDK - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: "11" - - name: Prepare JDK ${{ matrix.java_version }} - run: | - sudo apt-get -y update && sudo apt-get -y install curl - set -eux; - sudo mkdir -p /usr/lib/jvm/oracle8; - # https://gist.github.com/wavezhang/ba8425f24a968ec9b2a8619d7c2d86a6?permalink_comment_id=4444663#gistcomment-4444663 - # jdk1.8.0_361 - curl -L --fail "https://javadl.oracle.com/webapps/download/AutoDL?BundleId=247926_0ae14417abb444ebb02b9815e2103550" | sudo tar -xvzf - -C /usr/lib/jvm/oracle8 --strip-components 1 - uname -r - - name: Test - run: | - export TEST_COMMIT=${{ github.sha }} - export TEST_CONFIGURATION=glibc/8 - export LIBC=glibc - export JAVA_TEST_HOME=/usr/lib/jvm/oracle8 - export JAVA_HOME=$JAVA_HOME - export PATH=$JAVA_HOME/bin:$PATH - export BUILD_TYPE=SanitizedDebug - ./gradlew :ddprof-test:testDebug --info - - name: Upload logs - uses: actions/upload-artifact@v3 - if: always() - with: - name: reports-linux-oracle-jdk-8.zip - path: | - ddprof-test/hs_err_* - ddprof-test/build/reports/tests - ddprof-lib/src/test/build/Testing/Temporary/LastTest.log - ddprof-lib/build/tmp/compileReleaseLinuxCpp/output.txt - - uses: actions/upload-artifact@v3 - if: success() - with: - name: x64-oracle-jdk8 - path: build/ - - test-linux-musl: - needs: check-for-pr - if: needs.check-for-pr.outputs.skip != 'true' - strategy: - matrix: - java_version: [8u362+9, 11.0.18+10, 17.0.6+10, 21.0.1+12] - runs-on: ubuntu-latest - timeout-minutes: 10 - container: - image: "bellsoft/liberica-openjdk-alpine-musl:11.0.17" - options: --cpus 2 - steps: - - uses: actions/checkout@v3 - - name: Setup system - run: apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake >/dev/null - - name: Prepare build JDK - run: | - wget -nv https://download.bell-sw.com/java/11.0.18+10/bellsoft-jdk11.0.18+10-linux-x64-musl.tar.gz -O jdk.tar.gz - tar xzf *.tar.gz - find . -type d -name 'jdk*' -maxdepth 1 | xargs -I {} mv {} build_jdk - - name: Prepare JDK ${{ matrix.java_version }} - run: | - wget -nv https://download.bell-sw.com/java/${{ matrix.java_version }}/bellsoft-jdk${{ matrix.java_version }}-linux-x64-musl.tar.gz -O jdk.tar.gz - tar xzf *.tar.gz - find . -type d -name 'jdk*' -maxdepth 1 | xargs -I {} mv {} test_jdk - - name: Test - run: | - set -x - export JAVA_HOME=$(pwd)/build_jdk - export PATH=$JAVA_HOME/bin:$PATH - export TEST_COMMIT=${{ github.sha }} - export TEST_CONFIGURATION=musl/${{ matrix.java_version }} - export LIBC=musl - export JAVA_TEST_HOME=$(pwd)/test_jdk - ./gradlew :ddprof-test:testDebug --info - - name: Upload logs - uses: actions/upload-artifact@v3 - if: always() - with: - name: reports-linux-musl-${{ matrix.java_version }}.zip - path: | - ddprof-test/hs_err_* - ddprof-test/build/reports/tests - ddprof-lib/src/test/build/Testing/Temporary/LastTest.log - ddprof-lib/build/tmp/compileReleaseLinuxCpp/output.txt - - uses: actions/upload-artifact@v3 - if: success() - with: - name: x64-musl - path: build/ - - test-linux-glibc-zing: - needs: check-for-pr - if: needs.check-for-pr.outputs.skip != 'true' - strategy: - matrix: - java_version: [ 8, 11, 17, 21 ] - runs-on: ubuntu-latest - timeout-minutes: 180 - steps: - - uses: actions/checkout@v3 - - name: Prepare build JDK - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: "11" - - name: Prepare JDK ${{ matrix.java_version }} - run: | - sudo apt-get -y update && sudo apt-get -y install curl g++-9 gcc-9 - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9 - sudo update-alternatives --set gcc /usr/bin/gcc-9 - set -eux - sudo mkdir -p /usr/lib/jvm/zing - if [ "${{ matrix.java_version }}" = "8" ]; then - # jdk1.8.0_372 - curl -L --fail "https://cdn.azul.com/zing-zvm/ZVM23.05.0.0/zing23.05.0.0-2-jdk8.0.372-linux_x64.tar.gz" | sudo tar -xvzf - -C /usr/lib/jvm/zing --strip-components 1 - elif [ "${{ matrix.java_version }}" = "11" ]; then - # jdk 11.0.19 - curl -L --fail "https://cdn.azul.com/zing-zvm/ZVM23.05.0.0/zing23.05.0.0-2-jdk11.0.19-linux_x64.tar.gz" | sudo tar -xvzf - -C /usr/lib/jvm/zing --strip-components 1 - elif [ "${{ matrix.java_version }}" = "17" ]; then - # jdk 17.0.7 - curl -L --fail "https://cdn.azul.com/zing-zvm/ZVM23.05.0.0/zing23.05.0.0-2-jdk17.0.7-linux_x64.tar.gz" | sudo tar -xvzf - -C /usr/lib/jvm/zing --strip-components 1 - elif [ "${{ matrix.java_version }}" = "21" ]; then - # jdk 21.0.2 - curl -L --fail "https://cdn.azul.com/zing-zvm/ZVM23.10.0.0/zing23.10.0.0-3-jdk21.0.1-linux_x64.tar.gz" | sudo tar -xvzf - -C /usr/lib/jvm/zing --strip-components 1 - fi - # rename the bundled libstdc++.so to avoid conflicts with the system one - sudo mv /usr/lib/jvm/zing/etc/libc++/libstdc++.so.6 /usr/lib/jvm/zing/etc/libc++/libstdc++.so.6.bak - - name: Test - run: | - export TEST_COMMIT=${{ github.sha }} - export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} - export LIBC=glibc - export JAVA_TEST_HOME=/usr/lib/jvm/zing - export JAVA_HOME=$JAVA_HOME - export PATH=$JAVA_HOME/bin:$PATH - export BUILD_TYPE=SanitizedDebug - ./gradlew :ddprof-test:testDebug --info - - name: Upload logs - uses: actions/upload-artifact@v3 - if: always() - with: - name: reports-linux-zing-jdk-${{ matrix.java_version }}.zip - path: | - ddprof-test/hs_err_* - ddprof-test/build/reports/tests/test - ddprof-lib/src/test/build/Testing/Temporary/LastTest.log - ddprof-lib/build/tmp/compileReleaseLinuxCpp/output.txt - - uses: actions/upload-artifact@v3 - if: success() - with: - name: x64-glibc-zing-${{ matrix.java_version }} - path: build/ + uses: ./.github/workflows/test_workflow.yml + with: + configuration: '["debug"]' gh-release: if: startsWith(github.event.ref, 'refs/heads/release/') runs-on: ubuntu-latest - needs: [test-linux-musl, test-linux-glibc, test-linux-glibc-j9] + needs: [test-matrix] steps: - name: Create Github Release uses: ./.github/workflows/gh_release.yml@gh-release From cba18d5dae4e6dfa4bcd0e37cc25ba289788a3fe Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 17:51:56 +0200 Subject: [PATCH 46/80] Track failures in nightly --- .github/workflows/nightly.yml | 15 ++++++++++ .github/workflows/test_workflow.yml | 44 ++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 06b3eb3fe..baa434654 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -8,4 +8,19 @@ jobs: uses: ./.github/workflows/test_workflow.yml with: configuration: '["asan", "tsan"]' + report-failures: + runs-on: ubuntu-latest + needs: run-test + steps: + - name: Download failed tests artifact + uses: actions/download-artifact@v2 + with: + name: failures + path: ./artifacts + - name: Report failures + run: | + scenarios=$(cat ./artifacts/failures.txt | tr '\n' ',') + curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \ + -H 'Content-Type: application/json' \ + -d "{'scenarios': '${scenarios}', 'failed_run_url': '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'}" diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml index 2270e8616..b1398d276 100644 --- a/.github/workflows/test_workflow.yml +++ b/.github/workflows/test_workflow.yml @@ -32,7 +32,7 @@ jobs: export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} export LIBC=glibc export JAVA_TEST_HOME=$(pwd)/test_jdk - ./gradlew :ddprof-test:test${{ matrix.config }} --info + ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -48,6 +48,11 @@ jobs: with: name: x64-glibc-${{ matrix.java_version }}-${{ matrix.config }} path: build/ + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: failures + path: failures.txt test-ubuntu-jdk: strategy: @@ -76,7 +81,7 @@ jobs: export JAVA_TEST_HOME=/usr/lib/jvm/java-${{ matrix.java_version }}-openjdk-amd64 export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH - ./gradlew :ddprof-test:test${{ matrix.config }} --info + ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "ubuntu-jdk-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -92,6 +97,11 @@ jobs: with: name: x64-ubuntu-jdk-${{ matrix.java_version }}-${{ matrix.config }} path: build/ + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: failures + path: failures.txt test-linux-glibc-j9: strategy: @@ -120,7 +130,7 @@ jobs: export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} export LIBC=glibc chmod a+x gradlew - ./gradlew :ddprof-test:test${{ matrix.config }} --info + ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-j9-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -136,6 +146,11 @@ jobs: with: name: x64-glibc-j9-${{ matrix.java_version }}-${{ matrix.config }} path: build/ + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: failures + path: failures.txt test-linux-glibc-oracle8: strategy: @@ -167,7 +182,7 @@ jobs: export JAVA_TEST_HOME=/usr/lib/jvm/oracle8 export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH - ./gradlew :ddprof-test:test${{ matrix.config }} --info + ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-oracle8-${{ matrix.config }}" >> failures.txt - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -183,6 +198,11 @@ jobs: with: name: x64-oracle-jdk8-${{ matrix.config }} path: build/ + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: failures + path: failures.txt test-linux-musl: strategy: @@ -218,7 +238,7 @@ jobs: export TEST_CONFIGURATION=musl/${{ matrix.java_version }} export LIBC=musl export JAVA_TEST_HOME=$(pwd)/test_jdk - ./gradlew :ddprof-test:test${{ matrix.config }} --info + ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "musl-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -234,6 +254,11 @@ jobs: with: name: x64-musl-${{ matrix.java_version }}-${{ matrix.config }} path: build/ + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: failures + path: failures.txt test-linux-glibc-zing: strategy: @@ -280,7 +305,7 @@ jobs: export JAVA_TEST_HOME=/usr/lib/jvm/zing export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH - ./gradlew :ddprof-test:test${{ matrix.config }} --info + ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-zing-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -295,4 +320,9 @@ jobs: if: success() with: name: x64-glibc-zing-${{ matrix.java_version }}-${{ matrix.config }} - path: build/ \ No newline at end of file + path: build/ + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: failures + path: failures.txt \ No newline at end of file From 1109744a5744cf8ac1507bc777b6f39823c5a431 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 18:00:28 +0200 Subject: [PATCH 47/80] WIP --- .github/workflows/nightly.yml | 10 ++++-- .github/workflows/test_workflow.yml | 56 ++++++++++++++++++++++------- ddprof-lib/build.gradle | 30 +++++++++++++++- ddprof-lib/src/test/run_tests.sh | 7 ++-- ddprof-test/build.gradle | 4 +++ gradle/configurations.gradle | 30 ++++++++++++++-- 6 files changed, 116 insertions(+), 21 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index baa434654..5bbddf40c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,6 +11,7 @@ jobs: report-failures: runs-on: ubuntu-latest needs: run-test + if: failure() steps: - name: Download failed tests artifact uses: actions/download-artifact@v2 @@ -19,8 +20,11 @@ jobs: path: ./artifacts - name: Report failures run: | + find ./artifacts -name 'failures_*' -exec cat {} \; > ./artifacts/failures.txt scenarios=$(cat ./artifacts/failures.txt | tr '\n' ',') - curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \ - -H 'Content-Type: application/json' \ - -d "{'scenarios': '${scenarios}', 'failed_run_url': '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'}" + echo "Failed scenarios: $scenarios" + + #curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \ + # -H 'Content-Type: application/json' \ + # -d "{'scenarios': '${scenarios}', 'failed_run_url': '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'}" diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml index b1398d276..bb74add34 100644 --- a/.github/workflows/test_workflow.yml +++ b/.github/workflows/test_workflow.yml @@ -28,11 +28,16 @@ jobs: find . -type d -name 'jdk*' -maxdepth 1| xargs -I {} mv {} test_jdk - name: Test run: | + set +e export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} export LIBC=glibc export JAVA_TEST_HOME=$(pwd)/test_jdk - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "glibc-${{ matrix.java_version }}-${{ matrix.config }}" >> failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -52,7 +57,7 @@ jobs: if: failure() with: name: failures - path: failures.txt + path: failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}.txt test-ubuntu-jdk: strategy: @@ -75,13 +80,19 @@ jobs: uname -r - name: Test run: | + set +e + locate libtsan_preinit.o export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} export LIBC=glibc export JAVA_TEST_HOME=/usr/lib/jvm/java-${{ matrix.java_version }}-openjdk-amd64 export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "ubuntu-jdk-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "ubuntu-jdk-${{ matrix.java_version }}-${{ matrix.config }}" >> failures_ubuntu-jdk-${{ matrix.java_version }}-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -101,7 +112,7 @@ jobs: if: failure() with: name: failures - path: failures.txt + path: failures_ubuntu-jdk-${{ matrix.java_version }}-${{ matrix.config }}.txt test-linux-glibc-j9: strategy: @@ -126,11 +137,16 @@ jobs: java-version: "11" - name: Test run: | + set +e export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} export LIBC=glibc chmod a+x gradlew - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-j9-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "glibc-j9-${{ matrix.java_version }}-${{ matrix.config }}" >> failures_glibc-j9-${{ matrix.java_version }}-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -150,7 +166,7 @@ jobs: if: failure() with: name: failures - path: failures.txt + path: failures_glibc-j9-${{ matrix.java_version }}-${{ matrix.config }}.txt test-linux-glibc-oracle8: strategy: @@ -176,13 +192,18 @@ jobs: uname -r - name: Test run: | + set +e export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/8 export LIBC=glibc export JAVA_TEST_HOME=/usr/lib/jvm/oracle8 export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-oracle8-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "glibc-oracle8-${{ matrix.config }}" >> failures_glibc-oracle8-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -202,7 +223,7 @@ jobs: if: failure() with: name: failures - path: failures.txt + path: failures_glibc-oracle8-${{ matrix.config }}.txt test-linux-musl: strategy: @@ -231,14 +252,18 @@ jobs: find . -type d -name 'jdk*' -maxdepth 1 | xargs -I {} mv {} test_jdk - name: Test run: | - set -x + set +e export JAVA_HOME=$(pwd)/build_jdk export PATH=$JAVA_HOME/bin:$PATH export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=musl/${{ matrix.java_version }} export LIBC=musl export JAVA_TEST_HOME=$(pwd)/test_jdk - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "musl-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "musl-${{ matrix.java_version }}-${{ matrix.config }}" >> failures_musl-${{ matrix.java_version }}-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -258,7 +283,7 @@ jobs: if: failure() with: name: failures - path: failures.txt + path: failures_musl-${{ matrix.java_version }}-${{ matrix.config }}.txt test-linux-glibc-zing: strategy: @@ -299,13 +324,18 @@ jobs: sudo mv /usr/lib/jvm/zing/etc/libc++/libstdc++.so.6 /usr/lib/jvm/zing/etc/libc++/libstdc++.so.6.bak - name: Test run: | + set +e export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} export LIBC=glibc export JAVA_TEST_HOME=/usr/lib/jvm/zing export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-zing-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "glibc-zing-${{ matrix.java_version }}-${{ matrix.config }}" >> failures_zing-${{ matrix.java_version }}-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -325,4 +355,4 @@ jobs: if: failure() with: name: failures - path: failures.txt \ No newline at end of file + path: failures_zing-${{ matrix.java_version }}-${{ matrix.config }}.txt \ No newline at end of file diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index e378713f6..eeb94409c 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -112,6 +112,15 @@ buildConfigs().each { cfg -> }?.join(' ') def prepareGTestTask = tasks.register("prepareGTest${capitalizeFirstLetter(name)}", Exec) { + if (cfg.name == "asan") { + onlyIf { + locateLibasan() != null + } + } else if (cfg.name == "tsan") { + onlyIf { + locateLibtsan() != null + } + } def args = [] environment 'CMAKE_CXX_STANDARD', '11' if (osIdentifier() == 'macos') { @@ -131,8 +140,17 @@ buildConfigs().each { cfg -> outputs.dir file("${gtestDir}/configs/${name}") } def gtestTask = tasks.register("gtest${capitalizeFirstLetter(name)}", Exec) { + if (cfg.name == "asan") { + onlyIf { + locateLibasan() != null + } + } else if (cfg.name == "tsan") { + onlyIf { + locateLibtsan() != null + } + } onlyIf { - !project.hasProperty('skip-tests') && !project.hasProperty('skip-cpp-tests') + !project.hasProperty('skip-tests') && !project.hasProperty('skip-cpp-tests') && enabled } cfg.testEnv.each { key, value -> environment key, value @@ -141,6 +159,16 @@ buildConfigs().each { cfg -> workingDir projectDir commandLine "${projectDir}/src/test/run_tests.sh", name, projectDir, gtestDir, compilerFlags, osIdentifier() == 'linux' ? '-lrt -Wl,--no-as-needed' : '' outputs.upToDateWhen {true} + + // Capture the standard output and error if needed + standardOutput = new ByteArrayOutputStream() + errorOutput = new ByteArrayOutputStream() + + // Log the output for debugging purposes + doLast { + println "Standard Output: ${standardOutput.toString()}" + println "Error Output: ${errorOutput.toString()}" + } } } diff --git a/ddprof-lib/src/test/run_tests.sh b/ddprof-lib/src/test/run_tests.sh index 8b68943ea..4b4a70f55 100755 --- a/ddprof-lib/src/test/run_tests.sh +++ b/ddprof-lib/src/test/run_tests.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +set -euo pipefail +set -x + CONFIG_NAME=$1 PROJECT_DIR=$2 GTEST_DIR=$3 @@ -30,6 +33,6 @@ function build_and_run() { ${TMPDIR}/${NAME}_${CONFIG_NAME} } -build_and_run ddprof_ut -build_and_run demangle_ut +build_test ddprof_ut +build_test demangle_ut diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index f21313fd1..ff6356039 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -77,6 +77,10 @@ tasks.withType(Test).configureEach { onlyIf { hasAsan() } + } else if (name.contains("Tsan")) { + onlyIf { + hasTsan() + } } } diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index 8e3877dde..fbe91809d 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -21,6 +21,23 @@ def buildConfigs() { } } +def locate(def file) { + if (os().isLinux()) { + try { + def locateCommand = "locate ${file}" + def process = locateCommand.execute() + process.waitFor() + + if (process.exitValue() == 0) { + return process.in.text.trim() + } + } catch (Exception e) { + logger.warn("Exception when locating ${file}: ${e.message}") + } + } + return null +} + def locateLibrary(String libName) { if (os().isLinux()) { try { @@ -35,7 +52,7 @@ def locateLibrary(String libName) { } } } catch (Exception e) { - log.lifecycle("Exception when locating ${libName} library: ${e.message}") + logger.warn("Exception when locating ${libName} library: ${e.message}") } } return null @@ -46,7 +63,10 @@ def locateLibasan() { } def locateLibtsan() { - return locateLibrary('libtsan') + def libtsan_preinit = locate('libtsan_preinit.o') + if (libtsan_preinit != null && file(libtsan_preinit).exists()) { + return locateLibrary('libtsan') + } } def libasan = locateLibasan() @@ -61,6 +81,9 @@ if (libtsan == null) { def hasAsan() { return libasan != null } +def hasTsan() { + return libtsan != null +} ext.addBuildConfiguration = { String name, String os, String arch, List compilerArgs, List linkerArgs, Map testEnv = [:] -> buildConfigurations << [name: name, os: os, arch: arch, compilerArgs: compilerArgs, linkerArgs: linkerArgs, testEnv: testEnv] @@ -68,9 +91,12 @@ ext.addBuildConfiguration = { String name, String os, String arch, List ext { hasAsan = this.&hasAsan + hasTsan = this.&hasTsan capitalizeFirstLetter = this.&capitalizeFirstLetter buildConfigNames = this.&buildConfigNames buildConfigs = this.&buildConfigs + locateLibasan = this.&locateLibasan + locateLibtsan = this.&locateLibtsan } // ======= Define build configurations below ======== From f7949f9c2785ef64441e89c8da67826c8942bffa Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 28 May 2024 18:00:28 +0200 Subject: [PATCH 48/80] Improved CI tests --- .github/workflows/nightly.yml | 10 ++++-- .github/workflows/test_workflow.yml | 56 ++++++++++++++++++++++------- ddprof-lib/build.gradle | 30 +++++++++++++++- ddprof-lib/src/test/run_tests.sh | 7 ++-- ddprof-test/build.gradle | 15 ++++---- gradle/configurations.gradle | 30 ++++++++++++++-- 6 files changed, 121 insertions(+), 27 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index baa434654..5bbddf40c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,6 +11,7 @@ jobs: report-failures: runs-on: ubuntu-latest needs: run-test + if: failure() steps: - name: Download failed tests artifact uses: actions/download-artifact@v2 @@ -19,8 +20,11 @@ jobs: path: ./artifacts - name: Report failures run: | + find ./artifacts -name 'failures_*' -exec cat {} \; > ./artifacts/failures.txt scenarios=$(cat ./artifacts/failures.txt | tr '\n' ',') - curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \ - -H 'Content-Type: application/json' \ - -d "{'scenarios': '${scenarios}', 'failed_run_url': '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'}" + echo "Failed scenarios: $scenarios" + + #curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \ + # -H 'Content-Type: application/json' \ + # -d "{'scenarios': '${scenarios}', 'failed_run_url': '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'}" diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml index b1398d276..bb74add34 100644 --- a/.github/workflows/test_workflow.yml +++ b/.github/workflows/test_workflow.yml @@ -28,11 +28,16 @@ jobs: find . -type d -name 'jdk*' -maxdepth 1| xargs -I {} mv {} test_jdk - name: Test run: | + set +e export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} export LIBC=glibc export JAVA_TEST_HOME=$(pwd)/test_jdk - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "glibc-${{ matrix.java_version }}-${{ matrix.config }}" >> failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -52,7 +57,7 @@ jobs: if: failure() with: name: failures - path: failures.txt + path: failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}.txt test-ubuntu-jdk: strategy: @@ -75,13 +80,19 @@ jobs: uname -r - name: Test run: | + set +e + locate libtsan_preinit.o export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} export LIBC=glibc export JAVA_TEST_HOME=/usr/lib/jvm/java-${{ matrix.java_version }}-openjdk-amd64 export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "ubuntu-jdk-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "ubuntu-jdk-${{ matrix.java_version }}-${{ matrix.config }}" >> failures_ubuntu-jdk-${{ matrix.java_version }}-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -101,7 +112,7 @@ jobs: if: failure() with: name: failures - path: failures.txt + path: failures_ubuntu-jdk-${{ matrix.java_version }}-${{ matrix.config }}.txt test-linux-glibc-j9: strategy: @@ -126,11 +137,16 @@ jobs: java-version: "11" - name: Test run: | + set +e export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} export LIBC=glibc chmod a+x gradlew - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-j9-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "glibc-j9-${{ matrix.java_version }}-${{ matrix.config }}" >> failures_glibc-j9-${{ matrix.java_version }}-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -150,7 +166,7 @@ jobs: if: failure() with: name: failures - path: failures.txt + path: failures_glibc-j9-${{ matrix.java_version }}-${{ matrix.config }}.txt test-linux-glibc-oracle8: strategy: @@ -176,13 +192,18 @@ jobs: uname -r - name: Test run: | + set +e export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/8 export LIBC=glibc export JAVA_TEST_HOME=/usr/lib/jvm/oracle8 export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-oracle8-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "glibc-oracle8-${{ matrix.config }}" >> failures_glibc-oracle8-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -202,7 +223,7 @@ jobs: if: failure() with: name: failures - path: failures.txt + path: failures_glibc-oracle8-${{ matrix.config }}.txt test-linux-musl: strategy: @@ -231,14 +252,18 @@ jobs: find . -type d -name 'jdk*' -maxdepth 1 | xargs -I {} mv {} test_jdk - name: Test run: | - set -x + set +e export JAVA_HOME=$(pwd)/build_jdk export PATH=$JAVA_HOME/bin:$PATH export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=musl/${{ matrix.java_version }} export LIBC=musl export JAVA_TEST_HOME=$(pwd)/test_jdk - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "musl-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "musl-${{ matrix.java_version }}-${{ matrix.config }}" >> failures_musl-${{ matrix.java_version }}-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -258,7 +283,7 @@ jobs: if: failure() with: name: failures - path: failures.txt + path: failures_musl-${{ matrix.java_version }}-${{ matrix.config }}.txt test-linux-glibc-zing: strategy: @@ -299,13 +324,18 @@ jobs: sudo mv /usr/lib/jvm/zing/etc/libc++/libstdc++.so.6 /usr/lib/jvm/zing/etc/libc++/libstdc++.so.6.bak - name: Test run: | + set +e export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} export LIBC=glibc export JAVA_TEST_HOME=/usr/lib/jvm/zing export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH - ./gradlew :ddprof-test:test${{ matrix.config }} --info || echo "glibc-zing-${{ matrix.java_version }}-${{ matrix.config }}" >> failures.txt + ./gradlew :ddprof-test:test${{ matrix.config }} + if [ $? -ne 0 ]; then + echo "glibc-zing-${{ matrix.java_version }}-${{ matrix.config }}" >> failures_zing-${{ matrix.java_version }}-${{ matrix.config }}.txt + exit 1 + fi - name: Upload logs uses: actions/upload-artifact@v3 if: always() @@ -325,4 +355,4 @@ jobs: if: failure() with: name: failures - path: failures.txt \ No newline at end of file + path: failures_zing-${{ matrix.java_version }}-${{ matrix.config }}.txt \ No newline at end of file diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index e378713f6..eeb94409c 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -112,6 +112,15 @@ buildConfigs().each { cfg -> }?.join(' ') def prepareGTestTask = tasks.register("prepareGTest${capitalizeFirstLetter(name)}", Exec) { + if (cfg.name == "asan") { + onlyIf { + locateLibasan() != null + } + } else if (cfg.name == "tsan") { + onlyIf { + locateLibtsan() != null + } + } def args = [] environment 'CMAKE_CXX_STANDARD', '11' if (osIdentifier() == 'macos') { @@ -131,8 +140,17 @@ buildConfigs().each { cfg -> outputs.dir file("${gtestDir}/configs/${name}") } def gtestTask = tasks.register("gtest${capitalizeFirstLetter(name)}", Exec) { + if (cfg.name == "asan") { + onlyIf { + locateLibasan() != null + } + } else if (cfg.name == "tsan") { + onlyIf { + locateLibtsan() != null + } + } onlyIf { - !project.hasProperty('skip-tests') && !project.hasProperty('skip-cpp-tests') + !project.hasProperty('skip-tests') && !project.hasProperty('skip-cpp-tests') && enabled } cfg.testEnv.each { key, value -> environment key, value @@ -141,6 +159,16 @@ buildConfigs().each { cfg -> workingDir projectDir commandLine "${projectDir}/src/test/run_tests.sh", name, projectDir, gtestDir, compilerFlags, osIdentifier() == 'linux' ? '-lrt -Wl,--no-as-needed' : '' outputs.upToDateWhen {true} + + // Capture the standard output and error if needed + standardOutput = new ByteArrayOutputStream() + errorOutput = new ByteArrayOutputStream() + + // Log the output for debugging purposes + doLast { + println "Standard Output: ${standardOutput.toString()}" + println "Error Output: ${errorOutput.toString()}" + } } } diff --git a/ddprof-lib/src/test/run_tests.sh b/ddprof-lib/src/test/run_tests.sh index 8b68943ea..4b4a70f55 100755 --- a/ddprof-lib/src/test/run_tests.sh +++ b/ddprof-lib/src/test/run_tests.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +set -euo pipefail +set -x + CONFIG_NAME=$1 PROJECT_DIR=$2 GTEST_DIR=$3 @@ -30,6 +33,6 @@ function build_and_run() { ${TMPDIR}/${NAME}_${CONFIG_NAME} } -build_and_run ddprof_ut -build_and_run demangle_ut +build_test ddprof_ut +build_test demangle_ut diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index f21313fd1..4e9eee981 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -55,6 +55,15 @@ buildConfigurations.each { config -> } logger.debug("Setting environment variables for ${name}: ${config.testEnv}") } + if (config.name == "asan") { + onlyIf { + locateLibasan() != null + } + } else if (config.name == "tsan") { + onlyIf { + locateLibtsan() != null + } + } } } @@ -72,12 +81,6 @@ tasks.withType(Test).configureEach { } useJUnitPlatform() executable = new File("${javaHome}", 'bin/java') - - if (name.contains("Asan")) { - onlyIf { - hasAsan() - } - } } test { diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index 8e3877dde..fbe91809d 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -21,6 +21,23 @@ def buildConfigs() { } } +def locate(def file) { + if (os().isLinux()) { + try { + def locateCommand = "locate ${file}" + def process = locateCommand.execute() + process.waitFor() + + if (process.exitValue() == 0) { + return process.in.text.trim() + } + } catch (Exception e) { + logger.warn("Exception when locating ${file}: ${e.message}") + } + } + return null +} + def locateLibrary(String libName) { if (os().isLinux()) { try { @@ -35,7 +52,7 @@ def locateLibrary(String libName) { } } } catch (Exception e) { - log.lifecycle("Exception when locating ${libName} library: ${e.message}") + logger.warn("Exception when locating ${libName} library: ${e.message}") } } return null @@ -46,7 +63,10 @@ def locateLibasan() { } def locateLibtsan() { - return locateLibrary('libtsan') + def libtsan_preinit = locate('libtsan_preinit.o') + if (libtsan_preinit != null && file(libtsan_preinit).exists()) { + return locateLibrary('libtsan') + } } def libasan = locateLibasan() @@ -61,6 +81,9 @@ if (libtsan == null) { def hasAsan() { return libasan != null } +def hasTsan() { + return libtsan != null +} ext.addBuildConfiguration = { String name, String os, String arch, List compilerArgs, List linkerArgs, Map testEnv = [:] -> buildConfigurations << [name: name, os: os, arch: arch, compilerArgs: compilerArgs, linkerArgs: linkerArgs, testEnv: testEnv] @@ -68,9 +91,12 @@ ext.addBuildConfiguration = { String name, String os, String arch, List ext { hasAsan = this.&hasAsan + hasTsan = this.&hasTsan capitalizeFirstLetter = this.&capitalizeFirstLetter buildConfigNames = this.&buildConfigNames buildConfigs = this.&buildConfigs + locateLibasan = this.&locateLibasan + locateLibtsan = this.&locateLibtsan } // ======= Define build configurations below ======== From 966f0aa46d84402aa76c11e09e6d3b58302ceea4 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 29 May 2024 13:59:39 +0000 Subject: [PATCH 49/80] Migrate gtests to gradle --- ddprof-lib/build.gradle | 184 ++++++--------------------- ddprof-lib/gtest/build.gradle | 144 +++++++++++++++++++++ ddprof-lib/src/main/cpp/jvmHeap.h | 2 + ddprof-lib/src/test/prepare_gtest.sh | 29 ----- ddprof-lib/src/test/run_tests.sh | 38 ------ ddprof-test/build.gradle | 13 +- gradle/configurations.gradle | 25 +++- settings.gradle | 2 +- 8 files changed, 208 insertions(+), 229 deletions(-) create mode 100644 ddprof-lib/gtest/build.gradle delete mode 100755 ddprof-lib/src/test/prepare_gtest.sh delete mode 100755 ddprof-lib/src/test/run_tests.sh diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index eeb94409c..6afd67597 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -61,33 +61,6 @@ java { def isGitlabCI = System.getenv("GITLAB_CI") != null def buildTempDir = "${projectDir}/build/tmp" -def gtestVersion = "release-1.12.1" -def gtestDir = "${buildTempDir}/gtest" -def gtestLib = "${gtestDir}/build/lib/libgtest.a" - -tasks.register('downloadGTest', Download) { - src "https://github.com/google/googletest/archive/refs/tags/${gtestVersion}.zip" - dest "${buildTempDir}/gtest.zip" - overwrite false -} - -tasks.register('unzipGTest', Copy) { - dependsOn downloadGTest - from zipTree(tasks.downloadGTest.dest) - into "${gtestDir}" - eachFile { FileCopyDetails details -> - def pathSegments = details.relativePath.segments - if (pathSegments.length > 1) { - details.relativePath = new RelativePath(true, *pathSegments.drop(1)) - } else { - details.exclude() - } - } - doLast { - // take care of the dangling original top-level dir - file("${projectDir}/build/tmp/gtest/googletest-${gtestVersion}").deleteDir() - } -} // Allow specifying the external location for the native libraries // The libraries should be properly sorted into subfolders corresponding to the `libraryTargetPath` value for each @@ -104,76 +77,7 @@ tasks.register('copyExternalLibs', Copy) { tasks.register('assembleAll') {} -// gtest is 'specific' - we need to prepare it for each build configuration becuase it must be compiled with the same flags -buildConfigs().each { cfg -> - def name = cfg.name - def compilerFlags = cfg.compilerArgs.findAll { - return it.contains('sanitize') - }?.join(' ') - - def prepareGTestTask = tasks.register("prepareGTest${capitalizeFirstLetter(name)}", Exec) { - if (cfg.name == "asan") { - onlyIf { - locateLibasan() != null - } - } else if (cfg.name == "tsan") { - onlyIf { - locateLibtsan() != null - } - } - def args = [] - environment 'CMAKE_CXX_STANDARD', '11' - if (osIdentifier() == 'macos') { - args += "-DCMAKE_OSX_DEPLOYMENT_TARGET=10.10" - } - if (!compilerFlags.isEmpty()) { - environment "CMAKE_CXX_FLAGS", compilerFlags - environment "CMAKE_C_FLAGS", compilerFlags - environment "CMAKE_EXE_LINKER_FLAGS", compilerFlags - environment "CMAKE_SHARED_LINKER_FLAGS", compilerFlags - } - - dependsOn unzipGTest - workingDir gtestDir - commandLine "${projectDir}/src/test/prepare_gtest.sh", name, gtestDir, args.join(' ') - inputs.file file("${buildTempDir}/gtest.zip") - outputs.dir file("${gtestDir}/configs/${name}") - } - def gtestTask = tasks.register("gtest${capitalizeFirstLetter(name)}", Exec) { - if (cfg.name == "asan") { - onlyIf { - locateLibasan() != null - } - } else if (cfg.name == "tsan") { - onlyIf { - locateLibtsan() != null - } - } - onlyIf { - !project.hasProperty('skip-tests') && !project.hasProperty('skip-cpp-tests') && enabled - } - cfg.testEnv.each { key, value -> - environment key, value - } - dependsOn prepareGTestTask - workingDir projectDir - commandLine "${projectDir}/src/test/run_tests.sh", name, projectDir, gtestDir, compilerFlags, osIdentifier() == 'linux' ? '-lrt -Wl,--no-as-needed' : '' - outputs.upToDateWhen {true} - - // Capture the standard output and error if needed - standardOutput = new ByteArrayOutputStream() - errorOutput = new ByteArrayOutputStream() - - // Log the output for debugging purposes - doLast { - println "Standard Output: ${standardOutput.toString()}" - println "Error Output: ${errorOutput.toString()}" - } - } -} - // use the build config names to create configurations, copy lib and asemble jar tasks - buildConfigNames().each { name -> configurations.create(name) { canBeConsumed = true @@ -211,33 +115,45 @@ configurations { } } -// we need this trickery to reuse the toolchain and system config from tasks created by the cpp-library plugin +// We need this trickery to reuse the toolchain and system config from tasks created by the cpp-library plugin +// Basically, we are listening when the default 'comile' and 'link' (eg. 'compileReleaseCpp') is added and then +// we are adding our own tasks for each build configuration, inheriting the part of the configuration which was +// added by the cpp-library plugin tasks.whenTaskAdded { task -> if (task instanceof CppCompile) { if (!task.name.startsWith('compileLib') && task.name.contains('Release')) { buildConfigurations.each { config -> if (config.os == osIdentifier() && config.arch == archIdentifier()) { - def thisTask = tasks.register("compileLib${capitalizeFirstLetter(config.name)}", CppCompile) { + def cppTask = tasks.register("compileLib${capitalizeFirstLetter(config.name)}", CppCompile) { + onlyIf { + config.active + } group = 'build' description = "Compile the ${config.name} build of the library" objectFileDir = file("$buildDir/obj/main/${config.name}") compilerArgs.addAll(config.compilerArgs) + if (os().isLinux() && isMusl()) { + compilerArgs.add('-D__musl__') + } toolChain = task.toolChain targetPlatform = task.targetPlatform - includes = task.includes + includes task.includes + includes project(':ddprof-lib').file('src/main/cpp').toString() + includes "${javaHome()}/include" + includes project(':malloc-shim').file('src/main/public').toString() + if (os().isMacOsX()) { + includes "${javaHome()}/include/darwin" + } else if (os().isLinux()) { + includes "${javaHome()}/include/linux" + } systemIncludes = task.systemIncludes - source = task.source + source task.source inputs.files source outputs.dir objectFileDir } def linkTask = tasks.findByName("linkLib${capitalizeFirstLetter(config.name)}".toString()) if (linkTask != null) { - linkTask.dependsOn thisTask - } - def gtestTask = tasks.findByName("gtest${capitalizeFirstLetter(config.name)}".toString()) - if (gtestTask != null) { - gtestTask.dependsOn thisTask.get() - gtestTask.inputs.dir thisTask.get().objectFileDir + linkTask.dependsOn cppTask } } } @@ -246,7 +162,10 @@ tasks.whenTaskAdded { task -> if (!task.name.startsWith('linkLib') && task.name.contains('Release')) { buildConfigurations.each { config -> if (config.os == osIdentifier() && config.arch == archIdentifier()) { - def thisTask = tasks.register("linkLib${capitalizeFirstLetter(config.name)}", LinkSharedLibrary) { + def linkTask = tasks.register("linkLib${capitalizeFirstLetter(config.name)}", LinkSharedLibrary) { + onlyIf { + config.active + } group = 'build' description = "Link the ${config.name} build of the library" source = fileTree("$buildDir/obj/main/${config.name}") @@ -264,7 +183,7 @@ tasks.whenTaskAdded { task -> } if (config.name == 'release') { tasks.register('stripLibRelease', StripSymbols) { - dependsOn thisTask + dependsOn linkTask targetPlatform = tasks.linkLibRelease.targetPlatform toolChain = tasks.linkLibRelease.toolChain binaryFile = tasks.linkLibRelease.linkedFile.get() @@ -284,10 +203,6 @@ gradle.projectsEvaluated { buildConfigNames().each { def compileTask = tasks.findByName("compileLib${capitalizeFirstLetter(it)}") - def gtestTask = tasks.findByName("gtest${capitalizeFirstLetter(it)}") - if (compileTask != null && gtestTask != null) { - gtestTask.dependsOn compileTask - } def linkTask = tasks.findByName("linkLib${capitalizeFirstLetter(it)}") if (linkTask != null) { if (it != 'release') { @@ -296,58 +211,38 @@ gradle.projectsEvaluated { copyTask.dependsOn linkTask } } - linkTask.dependsOn gtestTask + def gtestTask = project(':ddprof-lib:gtest').tasks.findByName("gtest${capitalizeFirstLetter(it)}") + if (gtestTask != null) { + linkTask.dependsOn gtestTask + } } } } // configure the compiler here tasks.withType(CppCompile).configureEach { - if (name.startsWith('compileLib')) { - onlyIf { - !project.hasProperty('skip-native') - } - } else { + if (name.startsWith('compileRelease') || name.startsWith('compileDebug')) { onlyIf { // disable the built-in compiler task for release; we are using the custom compiler task false } - } - - def taskIncludes = ["${javaHome()}/include"] - def taskArgs = [] - - if (os().isMacOsX()) { - taskIncludes.add "${javaHome()}/include/darwin" - } else if (os().isLinux()) { - taskIncludes.add "${javaHome()}/include/linux" - if (isMusl()) { - taskArgs.add '-D__musl__' + } else { + onlyIf { + !project.hasProperty('skip-native') } } - - // TODO for Clang we would use scan-build utility instead but it needs to be installed first - - taskIncludes.add "${projectDir}/../malloc-shim/src/main/public" - - includes { - taskIncludes - } - - compilerArgs.addAll(taskArgs) - } // configure linker tasks.withType(LinkSharedLibrary).configureEach { - if (name.startsWith('linkLib')) { + if (name.startsWith('linkRelease') || name.startsWith('linkDebug')) { onlyIf { - !project.hasProperty('skip-native') + // disable the built-in linker task for release; we are using the custom linker task + false } } else { onlyIf { - // disable the built-in linker task for release; we are using the custom linker task - false + !project.hasProperty('skip-native') } } } @@ -374,6 +269,7 @@ dependencies { // the malloc shim works only on linux project(':malloc-shim') } + project(':ddprof-lib:gtest') } jar.dependsOn copyExternalLibs diff --git a/ddprof-lib/gtest/build.gradle b/ddprof-lib/gtest/build.gradle new file mode 100644 index 000000000..4c0b5374b --- /dev/null +++ b/ddprof-lib/gtest/build.gradle @@ -0,0 +1,144 @@ +plugins { + id 'cpp-application' +} + +// this feels weird but it is the only way invoking `./gradlew :ddprof-lib:*` tasks will work +if (rootDir.toString().endsWith("ddprof-lib/gradle")) { + apply from: rootProject.file('../../common.gradle') +} + +// disable the default compile and link tasks not to interfere with our custom ones +tasks.withType(CppCompile).configureEach { task -> + if (task.name.startsWith('compileRelease') || task.name.startsWith('compileDebug')) { + task.onlyIf { + false + } + } else { + onlyIf { + !project.hasProperty('skip-native') + } + } +} + +tasks.withType(LinkExecutable).configureEach { task -> + if (task.name.startsWith('linkRelease') || task.name.startsWith('linkDebug')) { + task.onlyIf { + false + } + } else { + onlyIf { + !project.hasProperty('skip-native') + } + } +} + +def gtestAll = tasks.register("gtest") { + group = 'verification' + description = "Run all Google Tests for all build configurations of the library" +} + +// we need this trickery to reuse the toolchain and system config from tasks created by the cpp-application plugin +tasks.whenTaskAdded { task -> + if (task instanceof CppCompile) { + if (!task.name.startsWith('compileGtest') && task.name.contains('Release')) { + buildConfigurations.each { config -> + if (config.os == osIdentifier() && config.arch == archIdentifier()) { + project(':ddprof-lib').file("src/test/cpp/").eachFile { + onlyIf { + config.active + } + def testFile = it + def testName = it.name.substring(0, it.name.lastIndexOf('.')) + def gtestCompileTask = tasks.register("compileGtest${capitalizeFirstLetter(config.name)}_${testName}", CppCompile) { + group = 'build' + description = "Compile the Google Test ${testName} for the ${config.name} build of the library" + objectFileDir = file("$buildDir/obj/gtest/${config.name}/${testName}") + compilerArgs.addAll(config.compilerArgs) + if (os().isLinux() && isMusl()) { + compilerArgs.add('-D__musl__') + } + toolChain = task.toolChain + targetPlatform = task.targetPlatform + includes task.includes + includes project(':ddprof-lib').file('src/main/cpp').toString() + includes "${javaHome()}/include" + includes project(':malloc-shim').file('src/main/public').toString() + if (os().isMacOsX()) { + includes "${javaHome()}/include/darwin" + } else if (os().isLinux()) { + includes "${javaHome()}/include/linux" + } + systemIncludes = task.systemIncludes + source project(':ddprof-lib').fileTree('src/main/cpp') { + include '**/*' + } + source testFile + + inputs.files source + outputs.dir objectFileDir + } + def linkTask = tasks.findByName("linkGtest${capitalizeFirstLetter(config.name)}_${testName}") + if (linkTask != null) { + linkTask.dependsOn gtestCompileTask + } + } + } + } + } + } else if (task instanceof LinkExecutable) { + if (!task.name.startsWith('linkGtest') && task.name.contains('Release')) { + buildConfigurations.each { config -> + if (config.os == osIdentifier() && config.arch == archIdentifier()) { + def gtestTask = tasks.register("gtest${capitalizeFirstLetter(config.name)}") { + group = 'verification' + description = "Run all Google Tests for the ${config.name} build of the library" + } + project(':ddprof-lib').file("src/test/cpp/").eachFile { + onlyIf { + config.active + } + def testFile = it + def testName = it.name.substring(0, it.name.lastIndexOf('.')) + def binary = file("$buildDir/bin/gtest/${config.name}_${testName}/${testName}") + def gtestLinkTask = tasks.register("linkGtest${capitalizeFirstLetter(config.name)}_${testName}", LinkExecutable) { + group = 'build' + description = "Link the Google Test for the ${config.name} build of the library" + source = fileTree("$buildDir/obj/gtest/${config.name}/${testName}") + linkedFile = binary + linkerArgs.addAll(config.linkerArgs) + linkerArgs.addAll("-lgtest", "-lgtest_main", "-lgmock", "-lgmock_main") + toolChain = task.toolChain + targetPlatform = task.targetPlatform + libs = task.libs + inputs.files source + outputs.file linkedFile + } + def gtestExecuteTask = tasks.register("gtest${capitalizeFirstLetter(config.name)}_${testName}", Exec) { + onlyIf { + config.active + } + group = 'verification' + description = "Run the Google Test ${testName} for the ${config.name} build of the library" + dependsOn gtestLinkTask + executable binary + + config.testEnv.each { key, value -> + environment key, value + } + + inputs.files binary + outputs.upToDateWhen {true} + } + + def compileTask = tasks.findByName("compileGtest${capitalizeFirstLetter(config.name)}_${testName}") + if (compileTask != null) { + gtestLinkTask.dependsOn compileTask + } + gtestTask.get().dependsOn gtestExecuteTask.get() + gtestAll.get().dependsOn gtestExecuteTask.get() + } + } + } + } + } +} \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/jvmHeap.h b/ddprof-lib/src/main/cpp/jvmHeap.h index 91114e22c..eafd8feb2 100644 --- a/ddprof-lib/src/main/cpp/jvmHeap.h +++ b/ddprof-lib/src/main/cpp/jvmHeap.h @@ -17,6 +17,8 @@ #ifndef _JVMHEAP_H #define _JVMHEAP_H +#include + /** * This class only defines a layout compatible with the JDKs VirtualSpaceSummary class and particularly its subclasses */ diff --git a/ddprof-lib/src/test/prepare_gtest.sh b/ddprof-lib/src/test/prepare_gtest.sh deleted file mode 100755 index be6c8ed6c..000000000 --- a/ddprof-lib/src/test/prepare_gtest.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -CONFIG_NAME=$1 -GTEST_DIR=$2 -shift -ARGS=$@ - -CFG_DIR="${GTEST_DIR}/configs/${CONFIG_NAME}" - -echo "Starting script with CONFIG_NAME=${CONFIG_NAME} and GTEST_DIR=${GTEST_DIR}" - -if [ ! -d "${CFG_DIR}" ]; then - mkdir -p "${CFG_DIR}" -fi -cd "${CFG_DIR}" -echo "Running cmake in ${CFG_DIR}" -cmake ../.. ${ARGS[@]} -DCMAKE_BUILD_TYPE=${CONFIG_NAME} -if [ $? -ne 0 ]; then - echo "cmake failed" - exit 1 -fi -echo "Running make in ${CFG_DIR}" -make -if [ $? -ne 0 ]; then - echo "make failed" - exit 1 -fi -echo "Build completed successfully." diff --git a/ddprof-lib/src/test/run_tests.sh b/ddprof-lib/src/test/run_tests.sh deleted file mode 100755 index 4b4a70f55..000000000 --- a/ddprof-lib/src/test/run_tests.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -set -x - -CONFIG_NAME=$1 -PROJECT_DIR=$2 -GTEST_DIR=$3 -shift -shift -shift -ARGS=$@ - -TMPDIR="${TMPDIR:-$(dirname $(mktemp))}" - -function build_test() { - NAME=$1 - echo "Building test ${NAME}..." - # shellcheck disable=SC2038 - files=$(find ${PROJECT_DIR}/build/obj/main/${CONFIG_NAME} -name "*.o" -print | tr '\n' ' ') - libdir=${GTEST_DIR}/configs/${CONFIG_NAME}/lib - g++ -std=c++11 -o ${TMPDIR}/${NAME}_${CONFIG_NAME} ${PROJECT_DIR}/src/test/cpp/${NAME}.cpp \ - -I${GTEST_DIR}/googletest/include -I${GTEST_DIR}/googlemock/include \ - -I${PROJECT_DIR}/src/main/cpp ${libdir}/libgtest.a \ - ${libdir}/libgtest_main.a ${libdir}/libgmock.a \ - ${libdir}/libgmock_main.a $files -lpthread -lm -ldl ${ARGS[@]} -pthread -} - -function build_and_run() { - NAME=$1 - build_test ${NAME} - echo "Running test ${NAME}..." - ${TMPDIR}/${NAME}_${CONFIG_NAME} -} - -build_test ddprof_ut -build_test demangle_ut - diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index ff6356039..c1eef8210 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -41,6 +41,9 @@ buildConfigurations.each { config -> cfg.dependencies.add(project.dependencies.project(path: ":ddprof-lib", configuration: name)) def task = tasks.register("test${name}", Test) { + onlyIf { + config.active + } dependsOn compileTestJava description = "Runs the unit tests with the ${name} library variant" group = 'verification' @@ -72,16 +75,6 @@ tasks.withType(Test).configureEach { } useJUnitPlatform() executable = new File("${javaHome}", 'bin/java') - - if (name.contains("Asan")) { - onlyIf { - hasAsan() - } - } else if (name.contains("Tsan")) { - onlyIf { - hasTsan() - } - } } test { diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index fbe91809d..5d2ff3f61 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -69,24 +69,35 @@ def locateLibtsan() { } } -def libasan = locateLibasan() -def libtsan = locateLibtsan() +ext.libasan = locateLibasan() +ext.libtsan = locateLibtsan() -if (libasan == null) { +if (ext.libasan == null) { logger.debug("Asan library not found") } -if (libtsan == null) { +if (ext.libtsan == null) { logger.debug("Tsan library not found") } + def hasAsan() { - return libasan != null + return ext.libasan != null } def hasTsan() { - return libtsan != null + return ext.libtsan != null +} + +def isActive(def name) { + def methodName = "has${name.capitalize()}" + if (this.metaClass.respondsTo(this, methodName)) { + this."${methodName}"() + } else { + // no activation method defined; default to 'true' + true + } } ext.addBuildConfiguration = { String name, String os, String arch, List compilerArgs, List linkerArgs, Map testEnv = [:] -> - buildConfigurations << [name: name, os: os, arch: arch, compilerArgs: compilerArgs, linkerArgs: linkerArgs, testEnv: testEnv] + buildConfigurations << [active: isActive(name), name: name, os: os, arch: arch, compilerArgs: compilerArgs, linkerArgs: linkerArgs, testEnv: testEnv] } ext { diff --git a/settings.gradle b/settings.gradle index b363c5606..7a5e35647 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,6 +1,6 @@ include ':ddprof-lib' +include ':ddprof-lib:gtest' include ':ddprof-test-tracer' include ':ddprof-test' include ':malloc-shim' include ':ddprof-stresstest' - From be360f9daa2eb745765f7de9133cd40131c4d846 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 29 May 2024 16:58:34 +0200 Subject: [PATCH 50/80] Make gtest tasks more flexible --- README.md | 5 ++-- common.gradle | 24 ++++++++++++----- ddprof-lib/build.gradle | 30 +++++++++------------ ddprof-lib/gtest/build.gradle | 49 +++++++++++++++++++---------------- ddprof-test/build.gradle | 2 +- gradle/configurations.gradle | 8 ------ 6 files changed, 60 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index c54f49ec1..b634d4c6a 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,11 @@ If you need a full-fledged Java profiler head back to [async-profiler](https://g ### Prerequisites 1. This project contains Java code and is built by gradle, as such, you need to install a JDK in order to build it. Run `brew install openjdk` to install it on MacOS. If building and testing on Apple Silicon (M1+), be sure to install an aarch64 OpenJDK distribution, otherwise the architecture used for compilation and the JVM based test suite will not match, which means the tests will not be able to load the dynamic library at runtime. -2. The build requires cmake, which is included in XCode, but can also be installed by running `brew install cmake` +2. The build requires make, which is included in XCode, but can also be installed by running `brew install make` +3. The build supports googletest for unit testing. To install it, run `brew install googletest` or `sudo apt-get install libgtest-dev` depending on your platform. -Once prerequisites have been installed simple as `./gradlew build`. The resulting artifact is located in `ddprof-lib/build/libs/ddprof-.jar` +Once prerequisites have been installed simple as `./gradlew assembleAll`. The resulting artifact is located in `ddprof-lib/build/libs/ddprof-.jar` diff --git a/common.gradle b/common.gradle index b48e3960b..776ff8e68 100644 --- a/common.gradle +++ b/common.gradle @@ -10,11 +10,11 @@ buildscript { } } -def os() { +def static os() { return org.gradle.internal.os.OperatingSystem.current() } -def osIdentifier() { +def static osIdentifier() { if (os().isMacOsX()) { return 'macos' } else if (os().isLinux()) { @@ -23,11 +23,12 @@ def osIdentifier() { throw new RuntimeException("Unknown OS: ${os().toString()}") } -def arch() { +def static arch() { return System.getProperty('os.arch') } -def archIdentifier() { +@SuppressWarnings('GroovyFallthrough') +def static archIdentifier() { String reportedArch = System.getProperty('os.arch') switch (reportedArch) { case 'x86_64': @@ -46,7 +47,7 @@ def archIdentifier() { } } -def javaHome() { +def static javaHome() { def jhome = System.getenv('JAVA_HOME') if (jhome == null) { jhome = System.getProperty('java.home') @@ -54,7 +55,7 @@ def javaHome() { return jhome } -def containsArray(byte[] container, int offset, byte[] contained) { +def static containsArray(byte[] container, int offset, byte[] contained) { for (int i = 0; i < contained.length; i++) { int leftPos = offset + i if (leftPos >= container.length) { @@ -74,7 +75,7 @@ def containsArray(byte[] container, int offset, byte[] contained) { * `/ld-musl` for musl systems and probably something else for non-musl systems (eg. `/ld-linux-...`). * However, if such string is missing should indicate that the system is not a musl one. */ -def isMusl() { +def static isMusl() { def magic = new byte[]{ 127, 69, 76, 70 } // *ELF @@ -107,6 +108,15 @@ def isMusl() { return false } +ext.hasGtest = false + +// This is hardcoded - we could have some discovery mechanism here but it would mean forking to shell +if (os().isMacOsX() && file('/opt/homebrew/opt/googletest').exists()) { + ext.hasGtest = true +} else if (os().isLinux() && file('/usr/include/gtest').exists()) { + ext.hasGtest = true +} + ext { isMusl = this.&isMusl os = this.&os diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 6afd67597..fb9a442ad 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -53,12 +53,6 @@ ext { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 -java { - toolchain { - languageVersion = JavaLanguageVersion.of(11) - } -} - def isGitlabCI = System.getenv("GITLAB_CI") != null def buildTempDir = "${projectDir}/build/tmp" @@ -84,11 +78,11 @@ buildConfigNames().each { name -> canBeResolved = false extendsFrom configurations.implementation } - def copyTask = tasks.register("copy${capitalizeFirstLetter(name)}Libs", Copy) { + def copyTask = tasks.register("copy${name.capitalize()}Libs", Copy) { from file(librarySourcePath(name, name == 'release' ? 'stripped' : '')).parent // the release build is stripped into file(libraryTargetPath(name)) } - def assembleTask = tasks.register("assemble${capitalizeFirstLetter(name)}Jar", Jar) { + def assembleTask = tasks.register("assemble${name.capitalize()}Jar", Jar) { group = 'build' description = "Assemble the ${name} build of the library" dependsOn copyExternalLibs @@ -124,7 +118,7 @@ tasks.whenTaskAdded { task -> if (!task.name.startsWith('compileLib') && task.name.contains('Release')) { buildConfigurations.each { config -> if (config.os == osIdentifier() && config.arch == archIdentifier()) { - def cppTask = tasks.register("compileLib${capitalizeFirstLetter(config.name)}", CppCompile) { + def cppTask = tasks.register("compileLib${config.name.capitalize()}", CppCompile) { onlyIf { config.active } @@ -151,7 +145,7 @@ tasks.whenTaskAdded { task -> inputs.files source outputs.dir objectFileDir } - def linkTask = tasks.findByName("linkLib${capitalizeFirstLetter(config.name)}".toString()) + def linkTask = tasks.findByName("linkLib${config.name.capitalize()}".toString()) if (linkTask != null) { linkTask.dependsOn cppTask } @@ -162,7 +156,7 @@ tasks.whenTaskAdded { task -> if (!task.name.startsWith('linkLib') && task.name.contains('Release')) { buildConfigurations.each { config -> if (config.os == osIdentifier() && config.arch == archIdentifier()) { - def linkTask = tasks.register("linkLib${capitalizeFirstLetter(config.name)}", LinkSharedLibrary) { + def linkTask = tasks.register("linkLib${config.name.capitalize()}", LinkSharedLibrary) { onlyIf { config.active } @@ -170,7 +164,7 @@ tasks.whenTaskAdded { task -> description = "Link the ${config.name} build of the library" source = fileTree("$buildDir/obj/main/${config.name}") linkedFile = file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/libjavaProfiler.${os().isLinux() ? 'so' : 'dylib'}") - def compileTask = tasks.findByName("compileLib${capitalizeFirstLetter(config.name)}".toString()) + def compileTask = tasks.findByName("compileLib${config.name.capitalize()}".toString()) if (compileTask != null) { dependsOn compileTask } @@ -202,16 +196,16 @@ gradle.projectsEvaluated { tasks.copyReleaseLibs.dependsOn tasks.stripLibRelease buildConfigNames().each { - def compileTask = tasks.findByName("compileLib${capitalizeFirstLetter(it)}") - def linkTask = tasks.findByName("linkLib${capitalizeFirstLetter(it)}") + def compileTask = tasks.findByName("compileLib${it.capitalize()}") + def linkTask = tasks.findByName("linkLib${it.capitalize()}") if (linkTask != null) { if (it != 'release') { - def copyTask = tasks.findByName("copy${capitalizeFirstLetter(it)}Libs") + def copyTask = tasks.findByName("copy${it.capitalize()}Libs") if (copyTask != null) { copyTask.dependsOn linkTask } } - def gtestTask = project(':ddprof-lib:gtest').tasks.findByName("gtest${capitalizeFirstLetter(it)}") + def gtestTask = project(':ddprof-lib:gtest').tasks.findByName("gtest${it.capitalize()}") if (gtestTask != null) { linkTask.dependsOn gtestTask } @@ -324,7 +318,7 @@ tasks.withType(Test) { artifacts { // create artifacts for all configures build config names buildConfigNames().each { - def task = tasks.named("assemble${capitalizeFirstLetter(it)}Jar") + def task = tasks.named("assemble${it.capitalize()}Jar") artifacts.add('assembled', task) artifacts.add(it, task) } @@ -334,7 +328,7 @@ publishing { publications { assembled(MavenPublication) { publication -> buildConfigNames().each { - publication.artifact tasks.named("assemble${capitalizeFirstLetter(it)}Jar") + publication.artifact tasks.named("assemble${it.capitalize()}Jar") } publication.artifact sourcesJar publication.artifact javadocJar diff --git a/ddprof-lib/gtest/build.gradle b/ddprof-lib/gtest/build.gradle index 4c0b5374b..6deeedece 100644 --- a/ddprof-lib/gtest/build.gradle +++ b/ddprof-lib/gtest/build.gradle @@ -13,10 +13,6 @@ tasks.withType(CppCompile).configureEach { task -> task.onlyIf { false } - } else { - onlyIf { - !project.hasProperty('skip-native') - } } } @@ -25,16 +21,19 @@ tasks.withType(LinkExecutable).configureEach { task -> task.onlyIf { false } - } else { - onlyIf { - !project.hasProperty('skip-native') - } } } def gtestAll = tasks.register("gtest") { + onlyIf { + hasGtest && !project.hasProperty('skip-native') + } group = 'verification' description = "Run all Google Tests for all build configurations of the library" + + if (!hasGtest) { + logger.warn("WARNING: Google Test not found - skipping native tests") + } } // we need this trickery to reuse the toolchain and system config from tasks created by the cpp-application plugin @@ -44,12 +43,12 @@ tasks.whenTaskAdded { task -> buildConfigurations.each { config -> if (config.os == osIdentifier() && config.arch == archIdentifier()) { project(':ddprof-lib').file("src/test/cpp/").eachFile { - onlyIf { - config.active - } def testFile = it def testName = it.name.substring(0, it.name.lastIndexOf('.')) - def gtestCompileTask = tasks.register("compileGtest${capitalizeFirstLetter(config.name)}_${testName}", CppCompile) { + def gtestCompileTask = tasks.register("compileGtest${config.name.capitalize()}_${testName}", CppCompile) { + onlyIf { + config.active && hasGtest && !project.hasProperty('skip-native') + } group = 'build' description = "Compile the Google Test ${testName} for the ${config.name} build of the library" objectFileDir = file("$buildDir/obj/gtest/${config.name}/${testName}") @@ -57,6 +56,7 @@ tasks.whenTaskAdded { task -> if (os().isLinux() && isMusl()) { compilerArgs.add('-D__musl__') } + compilerArgs.addAll('-std=c++14', '-fPIC') toolChain = task.toolChain targetPlatform = task.targetPlatform includes task.includes @@ -65,6 +65,7 @@ tasks.whenTaskAdded { task -> includes project(':malloc-shim').file('src/main/public').toString() if (os().isMacOsX()) { includes "${javaHome()}/include/darwin" + includes "/opt/homebrew/opt/googletest/include/" } else if (os().isLinux()) { includes "${javaHome()}/include/linux" } @@ -77,9 +78,9 @@ tasks.whenTaskAdded { task -> inputs.files source outputs.dir objectFileDir } - def linkTask = tasks.findByName("linkGtest${capitalizeFirstLetter(config.name)}_${testName}") + def linkTask = tasks.named("linkGtest${config.name.capitalize()}_${testName}") if (linkTask != null) { - linkTask.dependsOn gtestCompileTask + linkTask.get().dependsOn gtestCompileTask } } } @@ -89,33 +90,37 @@ tasks.whenTaskAdded { task -> if (!task.name.startsWith('linkGtest') && task.name.contains('Release')) { buildConfigurations.each { config -> if (config.os == osIdentifier() && config.arch == archIdentifier()) { - def gtestTask = tasks.register("gtest${capitalizeFirstLetter(config.name)}") { + def gtestTask = tasks.register("gtest${config.name.capitalize()}") { group = 'verification' description = "Run all Google Tests for the ${config.name} build of the library" } project(':ddprof-lib').file("src/test/cpp/").eachFile { - onlyIf { - config.active - } + def testFile = it def testName = it.name.substring(0, it.name.lastIndexOf('.')) def binary = file("$buildDir/bin/gtest/${config.name}_${testName}/${testName}") - def gtestLinkTask = tasks.register("linkGtest${capitalizeFirstLetter(config.name)}_${testName}", LinkExecutable) { + def gtestLinkTask = tasks.register("linkGtest${config.name.capitalize()}_${testName}", LinkExecutable) { + onlyIf { + config.active && hasGtest && !project.hasProperty('skip-native') + } group = 'build' description = "Link the Google Test for the ${config.name} build of the library" source = fileTree("$buildDir/obj/gtest/${config.name}/${testName}") linkedFile = binary linkerArgs.addAll(config.linkerArgs) linkerArgs.addAll("-lgtest", "-lgtest_main", "-lgmock", "-lgmock_main") + if (os().isMacOsX()) { + linkerArgs.addAll("-L/opt/homebrew/opt/googletest/lib") + } toolChain = task.toolChain targetPlatform = task.targetPlatform libs = task.libs inputs.files source outputs.file linkedFile } - def gtestExecuteTask = tasks.register("gtest${capitalizeFirstLetter(config.name)}_${testName}", Exec) { + def gtestExecuteTask = tasks.register("gtest${config.name.capitalize()}_${testName}", Exec) { onlyIf { - config.active + config.active && hasGtest && !project.hasProperty('skip-native') } group = 'verification' description = "Run the Google Test ${testName} for the ${config.name} build of the library" @@ -130,7 +135,7 @@ tasks.whenTaskAdded { task -> outputs.upToDateWhen {true} } - def compileTask = tasks.findByName("compileGtest${capitalizeFirstLetter(config.name)}_${testName}") + def compileTask = tasks.findByName("compileGtest${config.name.capitalize()}_${testName}") if (compileTask != null) { gtestLinkTask.dependsOn compileTask } diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index 5ff836dd2..871272cef 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -32,7 +32,7 @@ buildConfigurations.each { config -> } logger.debug("Creating configuration for ${name}") - def cfg = configurations.create("test${capitalizeFirstLetter(name)}Implementation") { + def cfg = configurations.create("test${name.capitalize()}Implementation") { canBeConsumed = true canBeResolved = true extendsFrom configurations.testCommon diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index c50e398c2..f906a3ed1 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -2,13 +2,6 @@ apply from: rootProject.file('common.gradle') ext.buildConfigurations = [] -def static capitalizeFirstLetter(str) { - if (str == null || str.isEmpty()) { - return str - } - return str.substring(0, 1).toUpperCase() + str.substring(1) -} - def buildConfigNames() { buildConfigurations.findAll { it.os == osIdentifier() && it.arch == archIdentifier() @@ -104,7 +97,6 @@ ext.addBuildConfiguration = { String name, String os, String arch, List ext { hasAsan = this.&hasAsan hasTsan = this.&hasTsan - capitalizeFirstLetter = this.&capitalizeFirstLetter buildConfigNames = this.&buildConfigNames buildConfigs = this.&buildConfigs locateLibasan = this.&locateLibasan From c175bf1ba8ed43822edcde8528832fef3e65ee00 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 29 May 2024 17:13:52 +0200 Subject: [PATCH 51/80] Fix the gtest failure for release --- ddprof-lib/gtest/build.gradle | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/gtest/build.gradle b/ddprof-lib/gtest/build.gradle index 6deeedece..01c8d42f1 100644 --- a/ddprof-lib/gtest/build.gradle +++ b/ddprof-lib/gtest/build.gradle @@ -52,11 +52,14 @@ tasks.whenTaskAdded { task -> group = 'build' description = "Compile the Google Test ${testName} for the ${config.name} build of the library" objectFileDir = file("$buildDir/obj/gtest/${config.name}/${testName}") - compilerArgs.addAll(config.compilerArgs) + compilerArgs.addAll(config.compilerArgs.findAll { + // need to drop the -std and -DNDEBUG flags because we need a different standard and assertions enabled + it != '-std=c++11' && it != '-DNDEBUG' + }) if (os().isLinux() && isMusl()) { compilerArgs.add('-D__musl__') } - compilerArgs.addAll('-std=c++14', '-fPIC') + compilerArgs.add('-std=c++14') toolChain = task.toolChain targetPlatform = task.targetPlatform includes task.includes From 6feecfb4929203c867019ad51bae7552cb410809 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 29 May 2024 15:36:42 +0000 Subject: [PATCH 52/80] Update readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b634d4c6a..366ffbdb4 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,8 @@ If you need a full-fledged Java profiler head back to [async-profiler](https://g 1. This project contains Java code and is built by gradle, as such, you need to install a JDK in order to build it. Run `brew install openjdk` to install it on MacOS. If building and testing on Apple Silicon (M1+), be sure to install an aarch64 OpenJDK distribution, otherwise the architecture used for compilation and the JVM based test suite will not match, which means the tests will not be able to load the dynamic library at runtime. 2. The build requires make, which is included in XCode, but can also be installed by running `brew install make` -3. The build supports googletest for unit testing. To install it, run `brew install googletest` or `sudo apt-get install libgtest-dev` depending on your platform. +3. The build may attempt to use `locate` to validate tsan installation. You might need to install it via `sudo apt install locate` +4. The build supports googletest for unit testing. To install it, run `brew install googletest` or `sudo apt install libgtest-dev` depending on your platform. Once prerequisites have been installed simple as `./gradlew assembleAll`. The resulting artifact is located in `ddprof-lib/build/libs/ddprof-.jar` From 4273b8891e9390f24329e164e84a7e494df7c20a Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 29 May 2024 17:44:06 +0200 Subject: [PATCH 53/80] Enable nightly run --- .github/workflows/nightly.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5bbddf40c..cdd246a4a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,6 +1,9 @@ name: Nightly Sanitized Run on: + schedule: + # Runs every day at 03:00 UTC + - cron: '0 3 * * *' workflow_dispatch: jobs: @@ -24,7 +27,7 @@ jobs: scenarios=$(cat ./artifacts/failures.txt | tr '\n' ',') echo "Failed scenarios: $scenarios" - #curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \ - # -H 'Content-Type: application/json' \ - # -d "{'scenarios': '${scenarios}', 'failed_run_url': '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'}" + curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \ + -H 'Content-Type: application/json' \ + -d "{'scenarios': '${scenarios}', 'failed_run_url': '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'}" From 83beece1880f9d1fb2a3d257cc159e105dbb7967 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 29 May 2024 17:26:21 +0000 Subject: [PATCH 54/80] Try to avoid read-only systemIncludes error --- ddprof-lib/build.gradle | 2 +- ddprof-lib/gtest/build.gradle | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index fb9a442ad..0e032ec5f 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -140,7 +140,7 @@ tasks.whenTaskAdded { task -> } else if (os().isLinux()) { includes "${javaHome()}/include/linux" } - systemIncludes = task.systemIncludes + includes task.systemIncludes source task.source inputs.files source outputs.dir objectFileDir diff --git a/ddprof-lib/gtest/build.gradle b/ddprof-lib/gtest/build.gradle index 01c8d42f1..b38a6db90 100644 --- a/ddprof-lib/gtest/build.gradle +++ b/ddprof-lib/gtest/build.gradle @@ -24,6 +24,18 @@ tasks.withType(LinkExecutable).configureEach { task -> } } +tasks.withType(ExtractSymbols).configureEach { task -> + task.onlyIf { + false + } +} + +tasks.withType(StripSymbols).configureEach { task -> + task.onlyIf { + false + } +} + def gtestAll = tasks.register("gtest") { onlyIf { hasGtest && !project.hasProperty('skip-native') @@ -72,7 +84,7 @@ tasks.whenTaskAdded { task -> } else if (os().isLinux()) { includes "${javaHome()}/include/linux" } - systemIncludes = task.systemIncludes + includes task.systemIncludes source project(':ddprof-lib').fileTree('src/main/cpp') { include '**/*' } From e346229493a807a5abab49d21e73753507ac4dc3 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 29 May 2024 17:52:57 +0000 Subject: [PATCH 55/80] Fix arm64 linux build --- ddprof-lib/build.gradle | 80 +++++++++++++++++++----------------- gradle/configurations.gradle | 23 +++++++++++ 2 files changed, 65 insertions(+), 38 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 0e032ec5f..4a5db7481 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -21,6 +21,21 @@ if (rootDir.toString().endsWith("ddprof-lib")) { apply from: rootProject.file('../common.gradle') } +dependencies { + if (os().isLinux()) { + // the malloc shim works only on linux + project(':malloc-shim') + } + project(':ddprof-lib:gtest') +} + +test { + onlyIf { + !project.hasProperty('skip-tests') + } + useJUnitPlatform() +} + def libraryTargetBase(type) { return "${projectDir}/build/native/${type}" } @@ -192,27 +207,6 @@ tasks.whenTaskAdded { task -> } } -gradle.projectsEvaluated { - tasks.copyReleaseLibs.dependsOn tasks.stripLibRelease - - buildConfigNames().each { - def compileTask = tasks.findByName("compileLib${it.capitalize()}") - def linkTask = tasks.findByName("linkLib${it.capitalize()}") - if (linkTask != null) { - if (it != 'release') { - def copyTask = tasks.findByName("copy${it.capitalize()}Libs") - if (copyTask != null) { - copyTask.dependsOn linkTask - } - } - def gtestTask = project(':ddprof-lib:gtest').tasks.findByName("gtest${it.capitalize()}") - if (gtestTask != null) { - linkTask.dependsOn gtestTask - } - } - } -} - // configure the compiler here tasks.withType(CppCompile).configureEach { if (name.startsWith('compileRelease') || name.startsWith('compileDebug')) { @@ -258,14 +252,6 @@ tasks.withType(StripSymbols).configureEach { } } -dependencies { - if (os().isLinux()) { - // the malloc shim works only on linux - project(':malloc-shim') - } - project(':ddprof-lib:gtest') -} - jar.dependsOn copyExternalLibs tasks.register('sourcesJar', Jar) { @@ -283,8 +269,6 @@ tasks.register('javadocJar', Jar) { from javadoc.destinationDir } -javadoc.dependsOn copyReleaseLibs - tasks.register('scanBuild', Exec) { workingDir "${projectDir}/src/test/make" commandLine 'scan-build' @@ -294,13 +278,6 @@ tasks.register('scanBuild', Exec) { "make", "-j4", "clean", "all" } -test { - onlyIf { - !project.hasProperty('skip-tests') - } - useJUnitPlatform() -} - tasks.withType(Test) { onlyIf { !project.hasProperty('skip-tests') @@ -315,6 +292,33 @@ tasks.withType(Test) { }) } +// relink the tasks when all are created +gradle.projectsEvaluated { + tasks.copyReleaseLibs.dependsOn tasks.stripLibRelease + + buildConfigNames().each { + def compileTask = tasks.findByName("compileLib${it.capitalize()}") + def linkTask = tasks.findByName("linkLib${it.capitalize()}") + if (linkTask != null) { + if (it != 'release') { + def copyTask = tasks.findByName("copy${it.capitalize()}Libs") + if (copyTask != null) { + copyTask.dependsOn linkTask + } + } + def gtestTask = project(':ddprof-lib:gtest').tasks.findByName("gtest${it.capitalize()}") + if (gtestTask != null) { + linkTask.dependsOn gtestTask + } + } + def javadocTask = tasks.findByName("javadoc") + def copyReleaseLibs = tasks.findByName("copyReleaseLibs") + if (javadocTask != null && copyReleaseLibs != null) { + javadocTask.dependsOn copyReleaseLibs + } + } +} + artifacts { // create artifacts for all configures build config names buildConfigNames().each { diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index f906a3ed1..1d501d637 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -216,6 +216,29 @@ addBuildConfiguration 'tsan', 'linux', 'x64', commonLinuxLinkerArgs + tsanLinkerArgs, tsanEnv +addBuildConfiguration 'release', 'linux', 'arm64', + ['-O3', '-DNDEBUG'] + commonLinuxCompilerArgs, + commonLinuxLinkerArgs + [ + "-Wl,-z,nodelete", + "-static-libstdc++", + "-static-libgcc", + "-Wl,--exclude-libs,ALL", + "-Wl,--gc-sections" + ] +addBuildConfiguration 'debug', 'linux', 'arm64', + ['-O0', '-g', '-DDEBUG'] + commonLinuxCompilerArgs, + commonLinuxLinkerArgs + +addBuildConfiguration 'asan', 'linux', 'arm64', + asanCompilerArgs + commonLinuxCompilerArgs, + commonLinuxLinkerArgs + asanLinkerArgs, + asanEnv + +addBuildConfiguration 'tsan', 'linux', 'arm64', + tsanCompilerArgs + commonLinuxCompilerArgs, + commonLinuxLinkerArgs + tsanLinkerArgs, + tsanEnv + // MacOS addBuildConfiguration 'release', 'macos', 'arm64', commonMacosCompilerArgs + ['-O3', '-DNDEBUG'], From 029951e5e09aba963b0334a66bc0546d4b6d5f9d Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 00:26:27 +0200 Subject: [PATCH 56/80] Aarch64 clang workaround --- ddprof-lib/build.gradle | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index fb9a442ad..0623fdbf7 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -140,7 +140,19 @@ tasks.whenTaskAdded { task -> } else if (os().isLinux()) { includes "${javaHome()}/include/linux" } - systemIncludes = task.systemIncludes + if (os().isLinux() && archIdentifier() == 'arm64') { + // aarch64 build is done in clang + // the clang binding for the non-built-in tasks is somehow messing up system includes + // so we need to add them manually + // this ties the build to clang 11 but there is nothing we can do about it + systemIncludes.from '/usr/include/c++/10' + systemIncludes.from '/usr/include/aarch64-linux-gnu/c++/10' + systemIncludes.from '/usr/include/c++/10/backward' + systemIncludes.from '/usr/local/include' + systemIncludes.from '/usr/lib/llvm-11/lib/clang/11.0.1/include' + systemIncludes.from '/usr/include/aarch64-linux-gnu' + systemIncludes.from '/usr/include' + } source task.source inputs.files source outputs.dir objectFileDir From ac782b69e5054f8fe91d3aadc2ef1b1ba253d8f3 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 11:23:50 +0200 Subject: [PATCH 57/80] Fix the release jar assembly --- ddprof-lib/build.gradle | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 592434730..a8b702e11 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -96,6 +96,13 @@ buildConfigNames().each { name -> def copyTask = tasks.register("copy${name.capitalize()}Libs", Copy) { from file(librarySourcePath(name, name == 'release' ? 'stripped' : '')).parent // the release build is stripped into file(libraryTargetPath(name)) + + if (name == 'release') { + def stripTask = tasks.findByName('stripLibRelease') + if (stripTask != null) { + dependsOn stripTask + } + } } def assembleTask = tasks.register("assemble${name.capitalize()}Jar", Jar) { group = 'build' @@ -203,7 +210,7 @@ tasks.whenTaskAdded { task -> outputs.file linkedFile } if (config.name == 'release') { - tasks.register('stripLibRelease', StripSymbols) { + def stripTask = tasks.register('stripLibRelease', StripSymbols) { dependsOn linkTask targetPlatform = tasks.linkLibRelease.targetPlatform toolChain = tasks.linkLibRelease.toolChain @@ -212,6 +219,10 @@ tasks.whenTaskAdded { task -> inputs.file binaryFile outputs.file outputFile } + def copyTask = tasks.findByName("copyReleaseLibs") + if (copyTask != null) { + copyTask.dependsOn stripTask + } } } } @@ -306,7 +317,7 @@ tasks.withType(Test) { // relink the tasks when all are created gradle.projectsEvaluated { - tasks.copyReleaseLibs.dependsOn tasks.stripLibRelease +// tasks.copyReleaseLibs.dependsOn tasks.stripLibRelease buildConfigNames().each { def compileTask = tasks.findByName("compileLib${it.capitalize()}") From d89b6496f15e5a860005721661080b002891b047 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 12:00:27 +0200 Subject: [PATCH 58/80] Fix assembly on aarch64 --- ddprof-lib/build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index a8b702e11..b42e10171 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -52,11 +52,11 @@ def osarchext() { } def libraryTargetPath(type) { - return "${libraryTargetBase(type)}/META-INF/native-libs/${osIdentifier()}-${osarchext()}${isMusl() ? '-musl' : ''}" + return "${libraryTargetBase(type)}/META-INF/native-libs/${osIdentifier()}-${archIdentifier()}${isMusl() ? '-musl' : ''}" } def librarySourcePath(type, qualifier = "") { - return "${projectDir}/build/lib/main/${type}/${osIdentifier()}/${osarchext()}/${qualifier}/libjavaProfiler.${osIdentifier() == 'macos' ? 'dylib' : 'so'}" + return "${projectDir}/build/lib/main/${type}/${osIdentifier()}/${archIdentifier()}/${qualifier}/libjavaProfiler.${osIdentifier() == 'macos' ? 'dylib' : 'so'}" } ext { @@ -222,6 +222,7 @@ tasks.whenTaskAdded { task -> def copyTask = tasks.findByName("copyReleaseLibs") if (copyTask != null) { copyTask.dependsOn stripTask + copyTask.inputs.files stripTask.get().outputs.files } } } From 1032728e81f89f9ca875b246795aae368d32edb3 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 12:16:43 +0200 Subject: [PATCH 59/80] Explicit sanitizers linker paths --- gradle/configurations.gradle | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index 1d501d637..3d89aa483 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -2,12 +2,14 @@ apply from: rootProject.file('common.gradle') ext.buildConfigurations = [] +@SuppressWarnings('GrMethodMayBeStatic') def buildConfigNames() { buildConfigurations.findAll { it.os == osIdentifier() && it.arch == archIdentifier() }.collect { it.name }.toSet() } +@SuppressWarnings('GrMethodMayBeStatic') def buildConfigs() { return buildConfigurations.findAll { it.os == osIdentifier() && it.arch == archIdentifier() @@ -121,7 +123,7 @@ def commonLinuxLinkerArgs = ["-ldl", "-Wl,-z,defs", "--verbose", "-lpthread", "- def commonMacosCompilerArgs = commonLinuxCompilerArgs + ["-D_XOPEN_SOURCE", "-D_DARWIN_C_SOURCE"] -def asanEnv = libasan != null ? +def asanEnv = hasAsan() ? ['LD_PRELOAD': libasan, // warning: stack use after return can cause slowness on arm64 "ASAN_OPTIONS" : "detect_stack_use_after_return=1 suppressions=${rootDir}/gradle/sanitizers/asan.supp", @@ -130,7 +132,7 @@ def asanEnv = libasan != null ? "LSAN_OPTIONS" : "detect_leaks=0" ] : [:] -def asanCompilerArgs = [ +def asanCompilerArgs = hasAsan() ? [ '-g', // Generate debug information '-DDEBUG', @@ -162,15 +164,18 @@ def asanCompilerArgs = [ '-fno-omit-frame-pointer', // Keep frame pointer for better debugging '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces -] +] : [] -def asanLinkerArgs = [ +def asanLinkerArgs = hasAsan() ? [ + "-L${file(libasan).parent}", + "-lasan", + "-lubsan", "-fsanitize=address", "-fsanitize=undefined", "-fno-omit-frame-pointer" // Often recommended with sanitizers -] +] : [] -def tsanCompilerArgs = [ +def tsanCompilerArgs = hasTsan() ? [ '-g', // Generate debug information '-DDEBUG', @@ -182,12 +187,16 @@ def tsanCompilerArgs = [ '-fno-omit-frame-pointer', // Keep frame pointer for better debugging '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces -] +] : [] -def tsanLinkerArgs = ["-fsanitize=thread", "-fno-omit-frame-pointer" // Often recommended with sanitizers -] +def tsanLinkerArgs = hasTsan() ? [ + "-L${file(libtsan).parent}", + "-ltsan", + "-fsanitize=thread", + "-fno-omit-frame-pointer" // Often recommended with sanitizers +] : [] -def tsanEnv = libtsan != null ? +def tsanEnv = hasTsan() ? ['LD_PRELOAD': libtsan, "TSAN_OPTIONS" : "suppressions=${rootDir}/gradle/sanitizers/tsan.supp" ] : [:] From 40d722016626d65b982bb5c179b62fe97a889f66 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 12:24:18 +0200 Subject: [PATCH 60/80] Try more systematic system includes --- ddprof-lib/build.gradle | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index b42e10171..52c23259d 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -162,19 +162,21 @@ tasks.whenTaskAdded { task -> } else if (os().isLinux()) { includes "${javaHome()}/include/linux" } - if (os().isLinux() && archIdentifier() == 'arm64') { - // aarch64 build is done in clang - // the clang binding for the non-built-in tasks is somehow messing up system includes - // so we need to add them manually - // this ties the build to clang 11 but there is nothing we can do about it - systemIncludes.from '/usr/include/c++/10' - systemIncludes.from '/usr/include/aarch64-linux-gnu/c++/10' - systemIncludes.from '/usr/include/c++/10/backward' - systemIncludes.from '/usr/local/include' - systemIncludes.from '/usr/lib/llvm-11/lib/clang/11.0.1/include' - systemIncludes.from '/usr/include/aarch64-linux-gnu' - systemIncludes.from '/usr/include' - } + // if (os().isLinux() && archIdentifier() == 'arm64') { + // // aarch64 build is done in clang + // // the clang binding for the non-built-in tasks is somehow messing up system includes + // // so we need to add them manually + // // this ties the build to clang 11 but there is nothing we can do about it + // systemIncludes.from '/usr/include/c++/10' + // systemIncludes.from '/usr/include/aarch64-linux-gnu/c++/10' + // systemIncludes.from '/usr/include/c++/10/backward' + // systemIncludes.from '/usr/local/include' + // systemIncludes.from '/usr/lib/llvm-11/lib/clang/11.0.1/include' + // systemIncludes.from '/usr/include/aarch64-linux-gnu' + // systemIncludes.from '/usr/include' + // } else { + systemIncludes.from task.systemIncludes + // } source task.source inputs.files source outputs.dir objectFileDir From 51db0110ab19f5c8484f694d4eb887e8a55e4ce1 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 13:25:43 +0200 Subject: [PATCH 61/80] Skip strip symmbols if 'skip-native' is specified --- ddprof-lib/build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 52c23259d..f890fd7b2 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -213,6 +213,9 @@ tasks.whenTaskAdded { task -> } if (config.name == 'release') { def stripTask = tasks.register('stripLibRelease', StripSymbols) { + onlyIf { + config.active + } dependsOn linkTask targetPlatform = tasks.linkLibRelease.targetPlatform toolChain = tasks.linkLibRelease.toolChain @@ -274,7 +277,7 @@ library { tasks.withType(StripSymbols).configureEach { onlyIf { - name == ("stripLibRelease") + name == ("stripLibRelease") && !project.hasProperty('skip-native') } } From 66ab531527a0e07b81eb893d593542dc0f67ab69 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 17:12:33 +0200 Subject: [PATCH 62/80] Upgrade Gradle to support JDK 22 --- gradle/wrapper/gradle-wrapper.jar | Bin 61608 -> 43453 bytes gradle/wrapper/gradle-wrapper.properties | 3 ++- gradlew | 29 +++++++++++++---------- gradlew.bat | 20 ++++++++-------- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index ccebba7710deaf9f98673a68957ea02138b60d0a..e6441136f3d4ba8a0da8d277868979cfbc8ad796 100644 GIT binary patch literal 43453 zcma&N1CXTcmMvW9vTb(Rwr$&4wr$(C?dmSu>@vG-+vuvg^_??!{yS%8zW-#zn-LkA z5&1^$^{lnmUON?}LBF8_K|(?T0Ra(xUH{($5eN!MR#ZihR#HxkUPe+_R8Cn`RRs(P z_^*#_XlXmGv7!4;*Y%p4nw?{bNp@UZHv1?Um8r6)Fei3p@ClJn0ECfg1hkeuUU@Or zDaPa;U3fE=3L}DooL;8f;P0ipPt0Z~9P0)lbStMS)ag54=uL9ia-Lm3nh|@(Y?B`; zx_#arJIpXH!U{fbCbI^17}6Ri*H<>OLR%c|^mh8+)*h~K8Z!9)DPf zR2h?lbDZQ`p9P;&DQ4F0sur@TMa!Y}S8irn(%d-gi0*WxxCSk*A?3lGh=gcYN?FGl z7D=Js!i~0=u3rox^eO3i@$0=n{K1lPNU zwmfjRVmLOCRfe=seV&P*1Iq=^i`502keY8Uy-WNPwVNNtJFx?IwAyRPZo2Wo1+S(xF37LJZ~%i)kpFQ3Fw=mXfd@>%+)RpYQLnr}B~~zoof(JVm^^&f zxKV^+3D3$A1G;qh4gPVjhrC8e(VYUHv#dy^)(RoUFM?o%W-EHxufuWf(l*@-l+7vt z=l`qmR56K~F|v<^Pd*p~1_y^P0P^aPC##d8+HqX4IR1gu+7w#~TBFphJxF)T$2WEa zxa?H&6=Qe7d(#tha?_1uQys2KtHQ{)Qco)qwGjrdNL7thd^G5i8Os)CHqc>iOidS} z%nFEDdm=GXBw=yXe1W-ShHHFb?Cc70+$W~z_+}nAoHFYI1MV1wZegw*0y^tC*s%3h zhD3tN8b=Gv&rj}!SUM6|ajSPp*58KR7MPpI{oAJCtY~JECm)*m_x>AZEu>DFgUcby z1Qaw8lU4jZpQ_$;*7RME+gq1KySGG#Wql>aL~k9tLrSO()LWn*q&YxHEuzmwd1?aAtI zBJ>P=&$=l1efe1CDU;`Fd+_;&wI07?V0aAIgc(!{a z0Jg6Y=inXc3^n!U0Atk`iCFIQooHqcWhO(qrieUOW8X(x?(RD}iYDLMjSwffH2~tB z)oDgNBLB^AJBM1M^c5HdRx6fBfka`(LD-qrlh5jqH~);#nw|iyp)()xVYak3;Ybik z0j`(+69aK*B>)e_p%=wu8XC&9e{AO4c~O1U`5X9}?0mrd*m$_EUek{R?DNSh(=br# z#Q61gBzEpmy`$pA*6!87 zSDD+=@fTY7<4A?GLqpA?Pb2z$pbCc4B4zL{BeZ?F-8`s$?>*lXXtn*NC61>|*w7J* z$?!iB{6R-0=KFmyp1nnEmLsA-H0a6l+1uaH^g%c(p{iT&YFrbQ$&PRb8Up#X3@Zsk zD^^&LK~111%cqlP%!_gFNa^dTYT?rhkGl}5=fL{a`UViaXWI$k-UcHJwmaH1s=S$4 z%4)PdWJX;hh5UoK?6aWoyLxX&NhNRqKam7tcOkLh{%j3K^4Mgx1@i|Pi&}<^5>hs5 zm8?uOS>%)NzT(%PjVPGa?X%`N2TQCKbeH2l;cTnHiHppPSJ<7y-yEIiC!P*ikl&!B z%+?>VttCOQM@ShFguHVjxX^?mHX^hSaO_;pnyh^v9EumqSZTi+#f&_Vaija0Q-e*| z7ulQj6Fs*bbmsWp{`auM04gGwsYYdNNZcg|ph0OgD>7O}Asn7^Z=eI>`$2*v78;sj-}oMoEj&@)9+ycEOo92xSyY344^ z11Hb8^kdOvbf^GNAK++bYioknrpdN>+u8R?JxG=!2Kd9r=YWCOJYXYuM0cOq^FhEd zBg2puKy__7VT3-r*dG4c62Wgxi52EMCQ`bKgf*#*ou(D4-ZN$+mg&7$u!! z-^+Z%;-3IDwqZ|K=ah85OLwkO zKxNBh+4QHh)u9D?MFtpbl)us}9+V!D%w9jfAMYEb>%$A;u)rrI zuBudh;5PN}_6J_}l55P3l_)&RMlH{m!)ai-i$g)&*M`eN$XQMw{v^r@-125^RRCF0 z^2>|DxhQw(mtNEI2Kj(;KblC7x=JlK$@78`O~>V!`|1Lm-^JR$-5pUANAnb(5}B}JGjBsliK4& zk6y(;$e&h)lh2)L=bvZKbvh@>vLlreBdH8No2>$#%_Wp1U0N7Ank!6$dFSi#xzh|( zRi{Uw%-4W!{IXZ)fWx@XX6;&(m_F%c6~X8hx=BN1&q}*( zoaNjWabE{oUPb!Bt$eyd#$5j9rItB-h*5JiNi(v^e|XKAj*8(k<5-2$&ZBR5fF|JA z9&m4fbzNQnAU}r8ab>fFV%J0z5awe#UZ|bz?Ur)U9bCIKWEzi2%A+5CLqh?}K4JHi z4vtM;+uPsVz{Lfr;78W78gC;z*yTch~4YkLr&m-7%-xc ztw6Mh2d>_iO*$Rd8(-Cr1_V8EO1f*^@wRoSozS) zy1UoC@pruAaC8Z_7~_w4Q6n*&B0AjOmMWa;sIav&gu z|J5&|{=a@vR!~k-OjKEgPFCzcJ>#A1uL&7xTDn;{XBdeM}V=l3B8fE1--DHjSaxoSjNKEM9|U9#m2<3>n{Iuo`r3UZp;>GkT2YBNAh|b z^jTq-hJp(ebZh#Lk8hVBP%qXwv-@vbvoREX$TqRGTgEi$%_F9tZES@z8Bx}$#5eeG zk^UsLBH{bc2VBW)*EdS({yw=?qmevwi?BL6*=12k9zM5gJv1>y#ML4!)iiPzVaH9% zgSImetD@dam~e>{LvVh!phhzpW+iFvWpGT#CVE5TQ40n%F|p(sP5mXxna+Ev7PDwA zamaV4m*^~*xV+&p;W749xhb_X=$|LD;FHuB&JL5?*Y2-oIT(wYY2;73<^#46S~Gx| z^cez%V7x$81}UWqS13Gz80379Rj;6~WdiXWOSsdmzY39L;Hg3MH43o*y8ibNBBH`(av4|u;YPq%{R;IuYow<+GEsf@R?=@tT@!}?#>zIIn0CoyV!hq3mw zHj>OOjfJM3F{RG#6ujzo?y32m^tgSXf@v=J$ELdJ+=5j|=F-~hP$G&}tDZsZE?5rX ztGj`!S>)CFmdkccxM9eGIcGnS2AfK#gXwj%esuIBNJQP1WV~b~+D7PJTmWGTSDrR` zEAu4B8l>NPuhsk5a`rReSya2nfV1EK01+G!x8aBdTs3Io$u5!6n6KX%uv@DxAp3F@{4UYg4SWJtQ-W~0MDb|j-$lwVn znAm*Pl!?Ps&3wO=R115RWKb*JKoexo*)uhhHBncEDMSVa_PyA>k{Zm2(wMQ(5NM3# z)jkza|GoWEQo4^s*wE(gHz?Xsg4`}HUAcs42cM1-qq_=+=!Gk^y710j=66(cSWqUe zklbm8+zB_syQv5A2rj!Vbw8;|$@C!vfNmNV!yJIWDQ>{+2x zKjuFX`~~HKG~^6h5FntRpnnHt=D&rq0>IJ9#F0eM)Y-)GpRjiN7gkA8wvnG#K=q{q z9dBn8_~wm4J<3J_vl|9H{7q6u2A!cW{bp#r*-f{gOV^e=8S{nc1DxMHFwuM$;aVI^ zz6A*}m8N-&x8;aunp1w7_vtB*pa+OYBw=TMc6QK=mbA-|Cf* zvyh8D4LRJImooUaSb7t*fVfih<97Gf@VE0|z>NcBwBQze);Rh!k3K_sfunToZY;f2 z^HmC4KjHRVg+eKYj;PRN^|E0>Gj_zagfRbrki68I^#~6-HaHg3BUW%+clM1xQEdPYt_g<2K+z!$>*$9nQ>; zf9Bei{?zY^-e{q_*|W#2rJG`2fy@{%6u0i_VEWTq$*(ZN37|8lFFFt)nCG({r!q#9 z5VK_kkSJ3?zOH)OezMT{!YkCuSSn!K#-Rhl$uUM(bq*jY? zi1xbMVthJ`E>d>(f3)~fozjg^@eheMF6<)I`oeJYx4*+M&%c9VArn(OM-wp%M<-`x z7sLP1&3^%Nld9Dhm@$3f2}87!quhI@nwd@3~fZl_3LYW-B?Ia>ui`ELg z&Qfe!7m6ze=mZ`Ia9$z|ARSw|IdMpooY4YiPN8K z4B(ts3p%2i(Td=tgEHX z0UQ_>URBtG+-?0E;E7Ld^dyZ;jjw0}XZ(}-QzC6+NN=40oDb2^v!L1g9xRvE#@IBR zO!b-2N7wVfLV;mhEaXQ9XAU+>=XVA6f&T4Z-@AX!leJ8obP^P^wP0aICND?~w&NykJ#54x3_@r7IDMdRNy4Hh;h*!u(Ol(#0bJdwEo$5437-UBjQ+j=Ic>Q2z` zJNDf0yO6@mr6y1#n3)s(W|$iE_i8r@Gd@!DWDqZ7J&~gAm1#~maIGJ1sls^gxL9LLG_NhU!pTGty!TbhzQnu)I*S^54U6Yu%ZeCg`R>Q zhBv$n5j0v%O_j{QYWG!R9W?5_b&67KB$t}&e2LdMvd(PxN6Ir!H4>PNlerpBL>Zvyy!yw z-SOo8caEpDt(}|gKPBd$qND5#a5nju^O>V&;f890?yEOfkSG^HQVmEbM3Ugzu+UtH zC(INPDdraBN?P%kE;*Ae%Wto&sgw(crfZ#Qy(<4nk;S|hD3j{IQRI6Yq|f^basLY; z-HB&Je%Gg}Jt@={_C{L$!RM;$$|iD6vu#3w?v?*;&()uB|I-XqEKqZPS!reW9JkLewLb!70T7n`i!gNtb1%vN- zySZj{8-1>6E%H&=V}LM#xmt`J3XQoaD|@XygXjdZ1+P77-=;=eYpoEQ01B@L*a(uW zrZeZz?HJsw_4g0vhUgkg@VF8<-X$B8pOqCuWAl28uB|@r`19DTUQQsb^pfqB6QtiT z*`_UZ`fT}vtUY#%sq2{rchyfu*pCg;uec2$-$N_xgjZcoumE5vSI{+s@iLWoz^Mf; zuI8kDP{!XY6OP~q5}%1&L}CtfH^N<3o4L@J@zg1-mt{9L`s^z$Vgb|mr{@WiwAqKg zp#t-lhrU>F8o0s1q_9y`gQNf~Vb!F%70f}$>i7o4ho$`uciNf=xgJ>&!gSt0g;M>*x4-`U)ysFW&Vs^Vk6m%?iuWU+o&m(2Jm26Y(3%TL; zA7T)BP{WS!&xmxNw%J=$MPfn(9*^*TV;$JwRy8Zl*yUZi8jWYF>==j~&S|Xinsb%c z2?B+kpet*muEW7@AzjBA^wAJBY8i|#C{WtO_or&Nj2{=6JTTX05}|H>N2B|Wf!*3_ z7hW*j6p3TvpghEc6-wufFiY!%-GvOx*bZrhZu+7?iSrZL5q9}igiF^*R3%DE4aCHZ zqu>xS8LkW+Auv%z-<1Xs92u23R$nk@Pk}MU5!gT|c7vGlEA%G^2th&Q*zfg%-D^=f z&J_}jskj|Q;73NP4<4k*Y%pXPU2Thoqr+5uH1yEYM|VtBPW6lXaetokD0u z9qVek6Q&wk)tFbQ8(^HGf3Wp16gKmr>G;#G(HRBx?F`9AIRboK+;OfHaLJ(P>IP0w zyTbTkx_THEOs%Q&aPrxbZrJlio+hCC_HK<4%f3ZoSAyG7Dn`=X=&h@m*|UYO-4Hq0 z-Bq&+Ie!S##4A6OGoC~>ZW`Y5J)*ouaFl_e9GA*VSL!O_@xGiBw!AF}1{tB)z(w%c zS1Hmrb9OC8>0a_$BzeiN?rkPLc9%&;1CZW*4}CDDNr2gcl_3z+WC15&H1Zc2{o~i) z)LLW=WQ{?ricmC`G1GfJ0Yp4Dy~Ba;j6ZV4r{8xRs`13{dD!xXmr^Aga|C=iSmor% z8hi|pTXH)5Yf&v~exp3o+sY4B^^b*eYkkCYl*T{*=-0HniSA_1F53eCb{x~1k3*`W zr~};p1A`k{1DV9=UPnLDgz{aJH=-LQo<5%+Em!DNN252xwIf*wF_zS^!(XSm(9eoj z=*dXG&n0>)_)N5oc6v!>-bd(2ragD8O=M|wGW z!xJQS<)u70m&6OmrF0WSsr@I%T*c#Qo#Ha4d3COcX+9}hM5!7JIGF>7<~C(Ear^Sn zm^ZFkV6~Ula6+8S?oOROOA6$C&q&dp`>oR-2Ym3(HT@O7Sd5c~+kjrmM)YmgPH*tL zX+znN>`tv;5eOfX?h{AuX^LK~V#gPCu=)Tigtq9&?7Xh$qN|%A$?V*v=&-2F$zTUv z`C#WyIrChS5|Kgm_GeudCFf;)!WH7FI60j^0o#65o6`w*S7R@)88n$1nrgU(oU0M9 zx+EuMkC>(4j1;m6NoGqEkpJYJ?vc|B zOlwT3t&UgL!pX_P*6g36`ZXQ; z9~Cv}ANFnJGp(;ZhS(@FT;3e)0)Kp;h^x;$*xZn*k0U6-&FwI=uOGaODdrsp-!K$Ac32^c{+FhI-HkYd5v=`PGsg%6I`4d9Jy)uW0y%) zm&j^9WBAp*P8#kGJUhB!L?a%h$hJgQrx!6KCB_TRo%9{t0J7KW8!o1B!NC)VGLM5! zpZy5Jc{`r{1e(jd%jsG7k%I+m#CGS*BPA65ZVW~fLYw0dA-H_}O zrkGFL&P1PG9p2(%QiEWm6x;U-U&I#;Em$nx-_I^wtgw3xUPVVu zqSuKnx&dIT-XT+T10p;yjo1Y)z(x1fb8Dzfn8e yu?e%!_ptzGB|8GrCfu%p?(_ zQccdaaVK$5bz;*rnyK{_SQYM>;aES6Qs^lj9lEs6_J+%nIiuQC*fN;z8md>r_~Mfl zU%p5Dt_YT>gQqfr@`cR!$NWr~+`CZb%dn;WtzrAOI>P_JtsB76PYe*<%H(y>qx-`Kq!X_; z<{RpAqYhE=L1r*M)gNF3B8r(<%8mo*SR2hu zccLRZwGARt)Hlo1euqTyM>^!HK*!Q2P;4UYrysje@;(<|$&%vQekbn|0Ruu_Io(w4#%p6ld2Yp7tlA`Y$cciThP zKzNGIMPXX%&Ud0uQh!uQZz|FB`4KGD?3!ND?wQt6!n*f4EmCoJUh&b?;B{|lxs#F- z31~HQ`SF4x$&v00@(P+j1pAaj5!s`)b2RDBp*PB=2IB>oBF!*6vwr7Dp%zpAx*dPr zb@Zjq^XjN?O4QcZ*O+8>)|HlrR>oD*?WQl5ri3R#2?*W6iJ>>kH%KnnME&TT@ZzrHS$Q%LC?n|e>V+D+8D zYc4)QddFz7I8#}y#Wj6>4P%34dZH~OUDb?uP%-E zwjXM(?Sg~1!|wI(RVuxbu)-rH+O=igSho_pDCw(c6b=P zKk4ATlB?bj9+HHlh<_!&z0rx13K3ZrAR8W)!@Y}o`?a*JJsD+twZIv`W)@Y?Amu_u zz``@-e2X}27$i(2=9rvIu5uTUOVhzwu%mNazS|lZb&PT;XE2|B&W1>=B58#*!~D&) zfVmJGg8UdP*fx(>Cj^?yS^zH#o-$Q-*$SnK(ZVFkw+er=>N^7!)FtP3y~Xxnu^nzY zikgB>Nj0%;WOltWIob|}%lo?_C7<``a5hEkx&1ku$|)i>Rh6@3h*`slY=9U}(Ql_< zaNG*J8vb&@zpdhAvv`?{=zDedJ23TD&Zg__snRAH4eh~^oawdYi6A3w8<Ozh@Kw)#bdktM^GVb zrG08?0bG?|NG+w^&JvD*7LAbjED{_Zkc`3H!My>0u5Q}m!+6VokMLXxl`Mkd=g&Xx z-a>m*#G3SLlhbKB!)tnzfWOBV;u;ftU}S!NdD5+YtOjLg?X}dl>7m^gOpihrf1;PY zvll&>dIuUGs{Qnd- zwIR3oIrct8Va^Tm0t#(bJD7c$Z7DO9*7NnRZorrSm`b`cxz>OIC;jSE3DO8`hX955ui`s%||YQtt2 z5DNA&pG-V+4oI2s*x^>-$6J?p=I>C|9wZF8z;VjR??Icg?1w2v5Me+FgAeGGa8(3S z4vg*$>zC-WIVZtJ7}o9{D-7d>zCe|z#<9>CFve-OPAYsneTb^JH!Enaza#j}^mXy1 z+ULn^10+rWLF6j2>Ya@@Kq?26>AqK{A_| zQKb*~F1>sE*=d?A?W7N2j?L09_7n+HGi{VY;MoTGr_)G9)ot$p!-UY5zZ2Xtbm=t z@dpPSGwgH=QtIcEulQNI>S-#ifbnO5EWkI;$A|pxJd885oM+ zGZ0_0gDvG8q2xebj+fbCHYfAXuZStH2j~|d^sBAzo46(K8n59+T6rzBwK)^rfPT+B zyIFw)9YC-V^rhtK`!3jrhmW-sTmM+tPH+;nwjL#-SjQPUZ53L@A>y*rt(#M(qsiB2 zx6B)dI}6Wlsw%bJ8h|(lhkJVogQZA&n{?Vgs6gNSXzuZpEyu*xySy8ro07QZ7Vk1!3tJphN_5V7qOiyK8p z#@jcDD8nmtYi1^l8ml;AF<#IPK?!pqf9D4moYk>d99Im}Jtwj6c#+A;f)CQ*f-hZ< z=p_T86jog%!p)D&5g9taSwYi&eP z#JuEK%+NULWus;0w32-SYFku#i}d~+{Pkho&^{;RxzP&0!RCm3-9K6`>KZpnzS6?L z^H^V*s!8<>x8bomvD%rh>Zp3>Db%kyin;qtl+jAv8Oo~1g~mqGAC&Qi_wy|xEt2iz zWAJEfTV%cl2Cs<1L&DLRVVH05EDq`pH7Oh7sR`NNkL%wi}8n>IXcO40hp+J+sC!W?!krJf!GJNE8uj zg-y~Ns-<~D?yqbzVRB}G>0A^f0!^N7l=$m0OdZuqAOQqLc zX?AEGr1Ht+inZ-Qiwnl@Z0qukd__a!C*CKuGdy5#nD7VUBM^6OCpxCa2A(X;e0&V4 zM&WR8+wErQ7UIc6LY~Q9x%Sn*Tn>>P`^t&idaOEnOd(Ufw#>NoR^1QdhJ8s`h^|R_ zXX`c5*O~Xdvh%q;7L!_!ohf$NfEBmCde|#uVZvEo>OfEq%+Ns7&_f$OR9xsihRpBb z+cjk8LyDm@U{YN>+r46?nn{7Gh(;WhFw6GAxtcKD+YWV?uge>;+q#Xx4!GpRkVZYu zzsF}1)7$?%s9g9CH=Zs+B%M_)+~*j3L0&Q9u7!|+T`^O{xE6qvAP?XWv9_MrZKdo& z%IyU)$Q95AB4!#hT!_dA>4e@zjOBD*Y=XjtMm)V|+IXzjuM;(l+8aA5#Kaz_$rR6! zj>#&^DidYD$nUY(D$mH`9eb|dtV0b{S>H6FBfq>t5`;OxA4Nn{J(+XihF(stSche7$es&~N$epi&PDM_N`As;*9D^L==2Q7Z2zD+CiU(|+-kL*VG+&9!Yb3LgPy?A zm7Z&^qRG_JIxK7-FBzZI3Q<;{`DIxtc48k> zc|0dmX;Z=W$+)qE)~`yn6MdoJ4co;%!`ddy+FV538Y)j(vg}5*k(WK)KWZ3WaOG!8 z!syGn=s{H$odtpqFrT#JGM*utN7B((abXnpDM6w56nhw}OY}0TiTG1#f*VFZr+^-g zbP10`$LPq_;PvrA1XXlyx2uM^mrjTzX}w{yuLo-cOClE8MMk47T25G8M!9Z5ypOSV zAJUBGEg5L2fY)ZGJb^E34R2zJ?}Vf>{~gB!8=5Z) z9y$>5c)=;o0HeHHSuE4U)#vG&KF|I%-cF6f$~pdYJWk_dD}iOA>iA$O$+4%@>JU08 zS`ep)$XLPJ+n0_i@PkF#ri6T8?ZeAot$6JIYHm&P6EB=BiaNY|aA$W0I+nz*zkz_z zkEru!tj!QUffq%)8y0y`T&`fuus-1p>=^hnBiBqD^hXrPs`PY9tU3m0np~rISY09> z`P3s=-kt_cYcxWd{de@}TwSqg*xVhp;E9zCsnXo6z z?f&Sv^U7n4`xr=mXle94HzOdN!2kB~4=%)u&N!+2;z6UYKUDqi-s6AZ!haB;@&B`? z_TRX0%@suz^TRdCb?!vNJYPY8L_}&07uySH9%W^Tc&1pia6y1q#?*Drf}GjGbPjBS zbOPcUY#*$3sL2x4v_i*Y=N7E$mR}J%|GUI(>WEr+28+V z%v5{#e!UF*6~G&%;l*q*$V?&r$Pp^sE^i-0$+RH3ERUUdQ0>rAq2(2QAbG}$y{de( z>{qD~GGuOk559Y@%$?N^1ApVL_a704>8OD%8Y%8B;FCt%AoPu8*D1 zLB5X>b}Syz81pn;xnB}%0FnwazlWfUV)Z-~rZg6~b z6!9J$EcE&sEbzcy?CI~=boWA&eeIa%z(7SE^qgVLz??1Vbc1*aRvc%Mri)AJaAG!p z$X!_9Ds;Zz)f+;%s&dRcJt2==P{^j3bf0M=nJd&xwUGlUFn?H=2W(*2I2Gdu zv!gYCwM10aeus)`RIZSrCK=&oKaO_Ry~D1B5!y0R=%!i2*KfXGYX&gNv_u+n9wiR5 z*e$Zjju&ODRW3phN925%S(jL+bCHv6rZtc?!*`1TyYXT6%Ju=|X;6D@lq$8T zW{Y|e39ioPez(pBH%k)HzFITXHvnD6hw^lIoUMA;qAJ^CU?top1fo@s7xT13Fvn1H z6JWa-6+FJF#x>~+A;D~;VDs26>^oH0EI`IYT2iagy23?nyJ==i{g4%HrAf1-*v zK1)~@&(KkwR7TL}L(A@C_S0G;-GMDy=MJn2$FP5s<%wC)4jC5PXoxrQBFZ_k0P{{s@sz+gX`-!=T8rcB(=7vW}^K6oLWMmp(rwDh}b zwaGGd>yEy6fHv%jM$yJXo5oMAQ>c9j`**}F?MCry;T@47@r?&sKHgVe$MCqk#Z_3S z1GZI~nOEN*P~+UaFGnj{{Jo@16`(qVNtbU>O0Hf57-P>x8Jikp=`s8xWs^dAJ9lCQ z)GFm+=OV%AMVqVATtN@|vp61VVAHRn87}%PC^RAzJ%JngmZTasWBAWsoAqBU+8L8u z4A&Pe?fmTm0?mK-BL9t+{y7o(7jm+RpOhL9KnY#E&qu^}B6=K_dB}*VlSEiC9fn)+V=J;OnN)Ta5v66ic1rG+dGAJ1 z1%Zb_+!$=tQ~lxQrzv3x#CPb?CekEkA}0MYSgx$Jdd}q8+R=ma$|&1a#)TQ=l$1tQ z=tL9&_^vJ)Pk}EDO-va`UCT1m#Uty1{v^A3P~83_#v^ozH}6*9mIjIr;t3Uv%@VeW zGL6(CwCUp)Jq%G0bIG%?{_*Y#5IHf*5M@wPo6A{$Um++Co$wLC=J1aoG93&T7Ho}P z=mGEPP7GbvoG!uD$k(H3A$Z))+i{Hy?QHdk>3xSBXR0j!11O^mEe9RHmw!pvzv?Ua~2_l2Yh~_!s1qS`|0~0)YsbHSz8!mG)WiJE| z2f($6TQtt6L_f~ApQYQKSb=`053LgrQq7G@98#igV>y#i==-nEjQ!XNu9 z~;mE+gtj4IDDNQJ~JVk5Ux6&LCSFL!y=>79kE9=V}J7tD==Ga+IW zX)r7>VZ9dY=V&}DR))xUoV!u(Z|%3ciQi_2jl}3=$Agc(`RPb z8kEBpvY>1FGQ9W$n>Cq=DIpski};nE)`p3IUw1Oz0|wxll^)4dq3;CCY@RyJgFgc# zKouFh!`?Xuo{IMz^xi-h=StCis_M7yq$u) z?XHvw*HP0VgR+KR6wI)jEMX|ssqYvSf*_3W8zVTQzD?3>H!#>InzpSO)@SC8q*ii- z%%h}_#0{4JG;Jm`4zg};BPTGkYamx$Xo#O~lBirRY)q=5M45n{GCfV7h9qwyu1NxOMoP4)jjZMxmT|IQQh0U7C$EbnMN<3)Kk?fFHYq$d|ICu>KbY_hO zTZM+uKHe(cIZfEqyzyYSUBZa8;Fcut-GN!HSA9ius`ltNebF46ZX_BbZNU}}ZOm{M2&nANL9@0qvih15(|`S~z}m&h!u4x~(%MAO$jHRWNfuxWF#B)E&g3ghSQ9|> z(MFaLQj)NE0lowyjvg8z0#m6FIuKE9lDO~Glg}nSb7`~^&#(Lw{}GVOS>U)m8bF}x zVjbXljBm34Cs-yM6TVusr+3kYFjr28STT3g056y3cH5Tmge~ASxBj z%|yb>$eF;WgrcOZf569sDZOVwoo%8>XO>XQOX1OyN9I-SQgrm;U;+#3OI(zrWyow3 zk==|{lt2xrQ%FIXOTejR>;wv(Pb8u8}BUpx?yd(Abh6? zsoO3VYWkeLnF43&@*#MQ9-i-d0t*xN-UEyNKeyNMHw|A(k(_6QKO=nKMCxD(W(Yop zsRQ)QeL4X3Lxp^L%wzi2-WVSsf61dqliPUM7srDB?Wm6Lzn0&{*}|IsKQW;02(Y&| zaTKv|`U(pSzuvR6Rduu$wzK_W-Y-7>7s?G$)U}&uK;<>vU}^^ns@Z!p+9?St1s)dG zK%y6xkPyyS1$~&6v{kl?Md6gwM|>mt6Upm>oa8RLD^8T{0?HC!Z>;(Bob7el(DV6x zi`I)$&E&ngwFS@bi4^xFLAn`=fzTC;aimE^!cMI2n@Vo%Ae-ne`RF((&5y6xsjjAZ zVguVoQ?Z9uk$2ON;ersE%PU*xGO@T*;j1BO5#TuZKEf(mB7|g7pcEA=nYJ{s3vlbg zd4-DUlD{*6o%Gc^N!Nptgay>j6E5;3psI+C3Q!1ZIbeCubW%w4pq9)MSDyB{HLm|k zxv-{$$A*pS@csolri$Ge<4VZ}e~78JOL-EVyrbxKra^d{?|NnPp86!q>t<&IP07?Z z^>~IK^k#OEKgRH+LjllZXk7iA>2cfH6+(e&9ku5poo~6y{GC5>(bRK7hwjiurqAiZ zg*DmtgY}v83IjE&AbiWgMyFbaRUPZ{lYiz$U^&Zt2YjG<%m((&_JUbZcfJ22(>bi5 z!J?<7AySj0JZ&<-qXX;mcV!f~>G=sB0KnjWca4}vrtunD^1TrpfeS^4dvFr!65knK zZh`d;*VOkPs4*-9kL>$GP0`(M!j~B;#x?Ba~&s6CopvO86oM?-? zOw#dIRc;6A6T?B`Qp%^<U5 z19x(ywSH$_N+Io!6;e?`tWaM$`=Db!gzx|lQ${DG!zb1Zl&|{kX0y6xvO1o z220r<-oaS^^R2pEyY;=Qllqpmue|5yI~D|iI!IGt@iod{Opz@*ml^w2bNs)p`M(Io z|E;;m*Xpjd9l)4G#KaWfV(t8YUn@A;nK^#xgv=LtnArX|vWQVuw3}B${h+frU2>9^ z!l6)!Uo4`5k`<<;E(ido7M6lKTgWezNLq>U*=uz&s=cc$1%>VrAeOoUtA|T6gO4>UNqsdK=NF*8|~*sl&wI=x9-EGiq*aqV!(VVXA57 zw9*o6Ir8Lj1npUXvlevtn(_+^X5rzdR>#(}4YcB9O50q97%rW2me5_L=%ffYPUSRc z!vv?Kv>dH994Qi>U(a<0KF6NH5b16enCp+mw^Hb3Xs1^tThFpz!3QuN#}KBbww`(h z7GO)1olDqy6?T$()R7y%NYx*B0k_2IBiZ14&8|JPFxeMF{vSTxF-Vi3+ZOI=Thq2} zyQgjYY1_7^ZQHh{?P))4+qUiQJLi1&{yE>h?~jU%tjdV0h|FENbM3X(KnJdPKc?~k zh=^Ixv*+smUll!DTWH!jrV*wSh*(mx0o6}1@JExzF(#9FXgmTXVoU+>kDe68N)dkQ zH#_98Zv$}lQwjKL@yBd;U(UD0UCl322=pav<=6g>03{O_3oKTq;9bLFX1ia*lw;#K zOiYDcBJf)82->83N_Y(J7Kr_3lE)hAu;)Q(nUVydv+l+nQ$?|%MWTy`t>{havFSQloHwiIkGK9YZ79^9?AZo0ZyQlVR#}lF%dn5n%xYksXf8gnBm=wO7g_^! zauQ-bH1Dc@3ItZ-9D_*pH}p!IG7j8A_o94#~>$LR|TFq zZ-b00*nuw|-5C2lJDCw&8p5N~Z1J&TrcyErds&!l3$eSz%`(*izc;-?HAFD9AHb-| z>)id`QCrzRws^9(#&=pIx9OEf2rmlob8sK&xPCWS+nD~qzU|qG6KwA{zbikcfQrdH z+ zQg>O<`K4L8rN7`GJB0*3<3`z({lWe#K!4AZLsI{%z#ja^OpfjU{!{)x0ZH~RB0W5X zTwN^w=|nA!4PEU2=LR05x~}|B&ZP?#pNgDMwD*ajI6oJqv!L81gu=KpqH22avXf0w zX3HjbCI!n9>l046)5rr5&v5ja!xkKK42zmqHzPx$9Nn_MZk`gLeSLgC=LFf;H1O#B zn=8|^1iRrujHfbgA+8i<9jaXc;CQBAmQvMGQPhFec2H1knCK2x!T`e6soyrqCamX% zTQ4dX_E*8so)E*TB$*io{$c6X)~{aWfaqdTh=xEeGvOAN9H&-t5tEE-qso<+C!2>+ zskX51H-H}#X{A75wqFe-J{?o8Bx|>fTBtl&tcbdR|132Ztqu5X0i-pisB-z8n71%q%>EF}yy5?z=Ve`}hVh{Drv1YWL zW=%ug_&chF11gDv3D6B)Tz5g54H0mDHNjuKZ+)CKFk4Z|$RD zfRuKLW`1B>B?*RUfVd0+u8h3r-{@fZ{k)c!93t1b0+Q9vOaRnEn1*IL>5Z4E4dZ!7 ztp4GP-^1d>8~LMeb}bW!(aAnB1tM_*la=Xx)q(I0Y@__Zd$!KYb8T2VBRw%e$iSdZ zkwdMwd}eV9q*;YvrBFTv1>1+}{H!JK2M*C|TNe$ZSA>UHKk);wz$(F$rXVc|sI^lD zV^?_J!3cLM;GJuBMbftbaRUs$;F}HDEDtIeHQ)^EJJ1F9FKJTGH<(Jj`phE6OuvE) zqK^K`;3S{Y#1M@8yRQwH`?kHMq4tHX#rJ>5lY3DM#o@or4&^_xtBC(|JpGTfrbGkA z2Tu+AyT^pHannww!4^!$5?@5v`LYy~T`qs7SYt$JgrY(w%C+IWA;ZkwEF)u5sDvOK zGk;G>Mh&elvXDcV69J_h02l&O;!{$({fng9Rlc3ID#tmB^FIG^w{HLUpF+iB`|
NnX)EH+Nua)3Y(c z&{(nX_ht=QbJ%DzAya}!&uNu!4V0xI)QE$SY__m)SAKcN0P(&JcoK*Lxr@P zY&P=}&B3*UWNlc|&$Oh{BEqwK2+N2U$4WB7Fd|aIal`FGANUa9E-O)!gV`((ZGCc$ zBJA|FFrlg~9OBp#f7aHodCe{6= zay$6vN~zj1ddMZ9gQ4p32(7wD?(dE>KA2;SOzXRmPBiBc6g`eOsy+pVcHu=;Yd8@{ zSGgXf@%sKKQz~;!J;|2fC@emm#^_rnO0esEn^QxXgJYd`#FPWOUU5b;9eMAF zZhfiZb|gk8aJIw*YLp4!*(=3l8Cp{(%p?ho22*vN9+5NLV0TTazNY$B5L6UKUrd$n zjbX%#m7&F#U?QNOBXkiiWB*_tk+H?N3`vg;1F-I+83{M2!8<^nydGr5XX}tC!10&e z7D36bLaB56WrjL&HiiMVtpff|K%|*{t*ltt^5ood{FOG0<>k&1h95qPio)2`eL${YAGIx(b4VN*~nKn6E~SIQUuRH zQ+5zP6jfnP$S0iJ@~t!Ai3o`X7biohli;E zT#yXyl{bojG@-TGZzpdVDXhbmF%F9+-^YSIv|MT1l3j zrxOFq>gd2%U}?6}8mIj?M zc077Zc9fq(-)4+gXv?Az26IO6eV`RAJz8e3)SC7~>%rlzDwySVx*q$ygTR5kW2ds- z!HBgcq0KON9*8Ff$X0wOq$`T7ml(@TF)VeoF}x1OttjuVHn3~sHrMB++}f7f9H%@f z=|kP_?#+fve@{0MlbkC9tyvQ_R?lRdRJ@$qcB(8*jyMyeME5ns6ypVI1Xm*Zr{DuS zZ!1)rQfa89c~;l~VkCiHI|PCBd`S*2RLNQM8!g9L6?n`^evQNEwfO@&JJRme+uopQX0%Jo zgd5G&#&{nX{o?TQwQvF1<^Cg3?2co;_06=~Hcb6~4XWpNFL!WU{+CK;>gH%|BLOh7@!hsa(>pNDAmpcuVO-?;Bic17R}^|6@8DahH)G z!EmhsfunLL|3b=M0MeK2vqZ|OqUqS8npxwge$w-4pFVXFq$_EKrZY?BuP@Az@(k`L z`ViQBSk`y+YwRT;&W| z2e3UfkCo^uTA4}Qmmtqs+nk#gNr2W4 zTH%hhErhB)pkXR{B!q5P3-OM+M;qu~f>}IjtF%>w{~K-0*jPVLl?Chz&zIdxp}bjx zStp&Iufr58FTQ36AHU)0+CmvaOpKF;W@sMTFpJ`j;3d)J_$tNQI^c<^1o<49Z(~K> z;EZTBaVT%14(bFw2ob@?JLQ2@(1pCdg3S%E4*dJ}dA*v}_a4_P(a`cHnBFJxNobAv zf&Zl-Yt*lhn-wjZsq<9v-IsXxAxMZ58C@e0!rzhJ+D@9^3~?~yllY^s$?&oNwyH!#~6x4gUrfxplCvK#!f z$viuszW>MFEcFL?>ux*((!L$;R?xc*myjRIjgnQX79@UPD$6Dz0jutM@7h_pq z0Zr)#O<^y_K6jfY^X%A-ip>P%3saX{!v;fxT-*0C_j4=UMH+Xth(XVkVGiiKE#f)q z%Jp=JT)uy{&}Iq2E*xr4YsJ5>w^=#-mRZ4vPXpI6q~1aFwi+lQcimO45V-JXP;>(Q zo={U`{=_JF`EQj87Wf}{Qy35s8r1*9Mxg({CvOt}?Vh9d&(}iI-quvs-rm~P;eRA@ zG5?1HO}puruc@S{YNAF3vmUc2B4!k*yi))<5BQmvd3tr}cIs#9)*AX>t`=~{f#Uz0 z0&Nk!7sSZwJe}=)-R^$0{yeS!V`Dh7w{w5rZ9ir!Z7Cd7dwZcK;BT#V0bzTt>;@Cl z#|#A!-IL6CZ@eHH!CG>OO8!%G8&8t4)Ro@}USB*k>oEUo0LsljsJ-%5Mo^MJF2I8- z#v7a5VdJ-Cd%(a+y6QwTmi+?f8Nxtm{g-+WGL>t;s#epv7ug>inqimZCVm!uT5Pf6 ziEgQt7^%xJf#!aPWbuC_3Nxfb&CFbQy!(8ANpkWLI4oSnH?Q3f?0k1t$3d+lkQs{~(>06l&v|MpcFsyAv zin6N!-;pggosR*vV=DO(#+}4ps|5$`udE%Kdmp?G7B#y%H`R|i8skKOd9Xzx8xgR$>Zo2R2Ytktq^w#ul4uicxW#{ zFjG_RNlBroV_n;a7U(KIpcp*{M~e~@>Q#Av90Jc5v%0c>egEdY4v3%|K1XvB{O_8G zkTWLC>OZKf;XguMH2-Pw{BKbFzaY;4v2seZV0>^7Q~d4O=AwaPhP3h|!hw5aqOtT@ z!SNz}$of**Bl3TK209@F=Tn1+mgZa8yh(Png%Zd6Mt}^NSjy)etQrF zme*llAW=N_8R*O~d2!apJnF%(JcN??=`$qs3Y+~xs>L9x`0^NIn!8mMRFA_tg`etw z3k{9JAjnl@ygIiJcNHTy02GMAvBVqEss&t2<2mnw!; zU`J)0>lWiqVqo|ex7!+@0i>B~BSU1A_0w#Ee+2pJx0BFiZ7RDHEvE*ptc9md(B{&+ zKE>TM)+Pd>HEmdJao7U@S>nL(qq*A)#eLOuIfAS@j`_sK0UEY6OAJJ-kOrHG zjHx`g!9j*_jRcJ%>CE9K2MVf?BUZKFHY?EpV6ai7sET-tqk=nDFh-(65rhjtlKEY% z@G&cQ<5BKatfdA1FKuB=i>CCC5(|9TMW%K~GbA4}80I5%B}(gck#Wlq@$nO3%@QP_ z8nvPkJFa|znk>V92cA!K1rKtr)skHEJD;k8P|R8RkCq1Rh^&}Evwa4BUJz2f!2=MH zo4j8Y$YL2313}H~F7@J7mh>u%556Hw0VUOz-Un@ZASCL)y8}4XXS`t1AC*^>PLwIc zUQok5PFS=*#)Z!3JZN&eZ6ZDP^-c@StY*t20JhCnbMxXf=LK#;`4KHEqMZ-Ly9KsS zI2VUJGY&PmdbM+iT)zek)#Qc#_i4uH43 z@T5SZBrhNCiK~~esjsO9!qBpaWK<`>!-`b71Y5ReXQ4AJU~T2Njri1CEp5oKw;Lnm)-Y@Z3sEY}XIgSy%xo=uek(kAAH5MsV$V3uTUsoTzxp_rF=tx zV07vlJNKtJhCu`b}*#m&5LV4TAE&%KtHViDAdv#c^x`J7bg z&N;#I2GkF@SIGht6p-V}`!F_~lCXjl1BdTLIjD2hH$J^YFN`7f{Q?OHPFEM$65^!u zNwkelo*5+$ZT|oQ%o%;rBX$+?xhvjb)SHgNHE_yP%wYkkvXHS{Bf$OiKJ5d1gI0j< zF6N}Aq=(WDo(J{e-uOecxPD>XZ@|u-tgTR<972`q8;&ZD!cep^@B5CaqFz|oU!iFj zU0;6fQX&~15E53EW&w1s9gQQ~Zk16X%6 zjG`j0yq}4deX2?Tr(03kg>C(!7a|b9qFI?jcE^Y>-VhudI@&LI6Qa}WQ>4H_!UVyF z((cm&!3gmq@;BD#5P~0;_2qgZhtJS|>WdtjY=q zLnHH~Fm!cxw|Z?Vw8*~?I$g#9j&uvgm7vPr#&iZgPP~v~BI4jOv;*OQ?jYJtzO<^y z7-#C={r7CO810!^s(MT!@@Vz_SVU)7VBi(e1%1rvS!?PTa}Uv`J!EP3s6Y!xUgM^8 z4f!fq<3Wer_#;u!5ECZ|^c1{|q_lh3m^9|nsMR1#Qm|?4Yp5~|er2?W^7~cl;_r4WSme_o68J9p03~Hc%X#VcX!xAu%1`R!dfGJCp zV*&m47>s^%Ib0~-2f$6oSgn3jg8m%UA;ArcdcRyM5;}|r;)?a^D*lel5C`V5G=c~k zy*w_&BfySOxE!(~PI$*dwG><+-%KT5p?whOUMA*k<9*gi#T{h3DAxzAPxN&Xws8o9Cp*`PA5>d9*Z-ynV# z9yY*1WR^D8|C%I@vo+d8r^pjJ$>eo|j>XiLWvTWLl(^;JHCsoPgem6PvegHb-OTf| zvTgsHSa;BkbG=(NgPO|CZu9gUCGr$8*EoH2_Z#^BnxF0yM~t`|9ws_xZ8X8iZYqh! zAh;HXJ)3P&)Q0(&F>!LN0g#bdbis-cQxyGn9Qgh`q+~49Fqd2epikEUw9caM%V6WgP)532RMRW}8gNS%V%Hx7apSz}tn@bQy!<=lbhmAH=FsMD?leawbnP5BWM0 z5{)@EEIYMu5;u)!+HQWhQ;D3_Cm_NADNeb-f56}<{41aYq8p4=93d=-=q0Yx#knGYfXVt z+kMxlus}t2T5FEyCN~!}90O_X@@PQpuy;kuGz@bWft%diBTx?d)_xWd_-(!LmVrh**oKg!1CNF&LX4{*j|) zIvjCR0I2UUuuEXh<9}oT_zT#jOrJAHNLFT~Ilh9hGJPI1<5`C-WA{tUYlyMeoy!+U zhA#=p!u1R7DNg9u4|QfED-2TuKI}>p#2P9--z;Bbf4Op*;Q9LCbO&aL2i<0O$ByoI z!9;Ght733FC>Pz>$_mw(F`zU?`m@>gE`9_p*=7o=7av`-&ifU(^)UU`Kg3Kw`h9-1 z6`e6+im=|m2v`pN(2dE%%n8YyQz;#3Q-|x`91z?gj68cMrHl}C25|6(_dIGk*8cA3 zRHB|Nwv{@sP4W+YZM)VKI>RlB`n=Oj~Rzx~M+Khz$N$45rLn6k1nvvD^&HtsMA4`s=MmuOJID@$s8Ph4E zAmSV^+s-z8cfv~Yd(40Sh4JG#F~aB>WFoX7ykaOr3JaJ&Lb49=B8Vk-SQT9%7TYhv z?-Pprt{|=Y5ZQ1?od|A<_IJU93|l4oAfBm?3-wk{O<8ea+`}u%(kub(LFo2zFtd?4 zwpN|2mBNywv+d^y_8#<$r>*5+$wRTCygFLcrwT(qc^n&@9r+}Kd_u@Ithz(6Qb4}A zWo_HdBj#V$VE#l6pD0a=NfB0l^6W^g`vm^sta>Tly?$E&{F?TTX~DsKF~poFfmN%2 z4x`Dc{u{Lkqz&y!33;X}weD}&;7p>xiI&ZUb1H9iD25a(gI|`|;G^NwJPv=1S5e)j z;U;`?n}jnY6rA{V^ zxTd{bK)Gi^odL3l989DQlN+Zs39Xe&otGeY(b5>rlIqfc7Ap4}EC?j<{M=hlH{1+d zw|c}}yx88_xQr`{98Z!d^FNH77=u(p-L{W6RvIn40f-BldeF-YD>p6#)(Qzf)lfZj z?3wAMtPPp>vMehkT`3gToPd%|D8~4`5WK{`#+}{L{jRUMt zrFz+O$C7y8$M&E4@+p+oV5c%uYzbqd2Y%SSgYy#xh4G3hQv>V*BnuKQhBa#=oZB~w{azUB+q%bRe_R^ z>fHBilnRTUfaJ201czL8^~Ix#+qOHSO)A|xWLqOxB$dT2W~)e-r9;bm=;p;RjYahB z*1hegN(VKK+ztr~h1}YP@6cfj{e#|sS`;3tJhIJK=tVJ-*h-5y9n*&cYCSdg#EHE# zSIx=r#qOaLJoVVf6v;(okg6?*L_55atl^W(gm^yjR?$GplNP>BZsBYEf_>wM0Lc;T zhf&gpzOWNxS>m+mN92N0{;4uw`P+9^*|-1~$uXpggj4- z^SFc4`uzj2OwdEVT@}Q`(^EcQ_5(ZtXTql*yGzdS&vrS_w>~~ra|Nb5abwf}Y!uq6R5f&6g2ge~2p(%c< z@O)cz%%rr4*cRJ5f`n@lvHNk@lE1a*96Kw6lJ~B-XfJW%?&-y?;E&?1AacU@`N`!O z6}V>8^%RZ7SQnZ-z$(jsX`amu*5Fj8g!3RTRwK^`2_QHe;_2y_n|6gSaGyPmI#kA0sYV<_qOZc#-2BO%hX)f$s-Z3xlI!ub z^;3ru11DA`4heAu%}HIXo&ctujzE2!6DIGE{?Zs>2}J+p&C$rc7gJC35gxhflorvsb%sGOxpuWhF)dL_&7&Z99=5M0b~Qa;Mo!j&Ti_kXW!86N%n= zSC@6Lw>UQ__F&+&Rzv?gscwAz8IP!n63>SP)^62(HK98nGjLY2*e^OwOq`3O|C92? z;TVhZ2SK%9AGW4ZavTB9?)mUbOoF`V7S=XM;#3EUpR+^oHtdV!GK^nXzCu>tpR|89 zdD{fnvCaN^^LL%amZ^}-E+214g&^56rpdc@yv0b<3}Ys?)f|fXN4oHf$six)-@<;W&&_kj z-B}M5U*1sb4)77aR=@%I?|Wkn-QJVuA96an25;~!gq(g1@O-5VGo7y&E_srxL6ZfS z*R%$gR}dyONgju*D&?geiSj7SZ@ftyA|}(*Y4KbvU!YLsi1EDQQCnb+-cM=K1io78o!v*);o<XwjaQH%)uIP&Zm?)Nfbfn;jIr z)d#!$gOe3QHp}2NBak@yYv3m(CPKkwI|{;d=gi552u?xj9ObCU^DJFQp4t4e1tPzM zvsRIGZ6VF+{6PvqsplMZWhz10YwS={?`~O0Ec$`-!klNUYtzWA^f9m7tkEzCy<_nS z=&<(awFeZvt51>@o_~>PLs05CY)$;}Oo$VDO)?l-{CS1Co=nxjqben*O1BR>#9`0^ zkwk^k-wcLCLGh|XLjdWv0_Hg54B&OzCE^3NCP}~OajK-LuRW53CkV~Su0U>zN%yQP zH8UH#W5P3-!ToO-2k&)}nFe`t+mdqCxxAHgcifup^gKpMObbox9LFK;LP3}0dP-UW z?Zo*^nrQ6*$FtZ(>kLCc2LY*|{!dUn$^RW~m9leoF|@Jy|M5p-G~j%+P0_#orRKf8 zvuu5<*XO!B?1E}-*SY~MOa$6c%2cM+xa8}_8x*aVn~57v&W(0mqN1W`5a7*VN{SUH zXz98DDyCnX2EPl-`Lesf`=AQT%YSDb`$%;(jUTrNen$NPJrlpPDP}prI>Ml!r6bCT;mjsg@X^#&<}CGf0JtR{Ecwd&)2zuhr#nqdgHj+g2n}GK9CHuwO zk>oZxy{vcOL)$8-}L^iVfJHAGfwN$prHjYV0ju}8%jWquw>}_W6j~m<}Jf!G?~r5&Rx)!9JNX!ts#SGe2HzobV5); zpj@&`cNcO&q+%*<%D7za|?m5qlmFK$=MJ_iv{aRs+BGVrs)98BlN^nMr{V_fcl_;jkzRju+c-y?gqBC_@J0dFLq-D9@VN&-`R9U;nv$Hg?>$oe4N&Ht$V_(JR3TG^! zzJsbQbi zFE6-{#9{G{+Z}ww!ycl*7rRdmU#_&|DqPfX3CR1I{Kk;bHwF6jh0opI`UV2W{*|nn zf_Y@%wW6APb&9RrbEN=PQRBEpM(N1w`81s=(xQj6 z-eO0k9=Al|>Ej|Mw&G`%q8e$2xVz1v4DXAi8G};R$y)ww638Y=9y$ZYFDM$}vzusg zUf+~BPX>(SjA|tgaFZr_e0{)+z9i6G#lgt=F_n$d=beAt0Sa0a7>z-?vcjl3e+W}+ z1&9=|vC=$co}-Zh*%3588G?v&U7%N1Qf-wNWJ)(v`iO5KHSkC5&g7CrKu8V}uQGcfcz zmBz#Lbqwqy#Z~UzHgOQ;Q-rPxrRNvl(&u6ts4~0=KkeS;zqURz%!-ERppmd%0v>iRlEf+H$yl{_8TMJzo0 z>n)`On|7=WQdsqhXI?#V{>+~}qt-cQbokEbgwV3QvSP7&hK4R{Z{aGHVS3;+h{|Hz z6$Js}_AJr383c_+6sNR|$qu6dqHXQTc6?(XWPCVZv=)D#6_;D_8P-=zOGEN5&?~8S zl5jQ?NL$c%O)*bOohdNwGIKM#jSAC?BVY={@A#c9GmX0=T(0G}xs`-%f3r=m6-cpK z!%waekyAvm9C3%>sixdZj+I(wQlbB4wv9xKI*T13DYG^T%}zZYJ|0$Oj^YtY+d$V$ zAVudSc-)FMl|54n=N{BnZTM|!>=bhaja?o7s+v1*U$!v!qQ%`T-6fBvmdPbVmro&d zk07TOp*KuxRUSTLRrBj{mjsnF8`d}rMViY8j`jo~Hp$fkv9F_g(jUo#Arp;Xw0M$~ zRIN!B22~$kx;QYmOkos@%|5k)!QypDMVe}1M9tZfkpXKGOxvKXB!=lo`p?|R1l=tA zp(1}c6T3Fwj_CPJwVsYtgeRKg?9?}%oRq0F+r+kdB=bFUdVDRPa;E~~>2$w}>O>v=?|e>#(-Lyx?nbg=ckJ#5U6;RT zNvHhXk$P}m9wSvFyU3}=7!y?Y z=fg$PbV8d7g25&-jOcs{%}wTDKm>!Vk);&rr;O1nvO0VrU&Q?TtYVU=ir`te8SLlS zKSNmV=+vF|ATGg`4$N1uS|n??f}C_4Sz!f|4Ly8#yTW-FBfvS48Tef|-46C(wEO_%pPhUC5$-~Y?!0vFZ^Gu`x=m7X99_?C-`|h zfmMM&Y@zdfitA@KPw4Mc(YHcY1)3*1xvW9V-r4n-9ZuBpFcf{yz+SR{ zo$ZSU_|fgwF~aakGr(9Be`~A|3)B=9`$M-TWKipq-NqRDRQc}ABo*s_5kV%doIX7LRLRau_gd@Rd_aLFXGSU+U?uAqh z8qusWWcvgQ&wu{|sRXmv?sl=xc<$6AR$+cl& zFNh5q1~kffG{3lDUdvEZu5c(aAG~+64FxdlfwY^*;JSS|m~CJusvi-!$XR`6@XtY2 znDHSz7}_Bx7zGq-^5{stTRy|I@N=>*y$zz>m^}^{d&~h;0kYiq8<^Wq7Dz0w31ShO^~LUfW6rfitR0(=3;Uue`Y%y@ex#eKPOW zO~V?)M#AeHB2kovn1v=n^D?2{2jhIQd9t|_Q+c|ZFaWt+r&#yrOu-!4pXAJuxM+Cx z*H&>eZ0v8Y`t}8{TV6smOj=__gFC=eah)mZt9gwz>>W$!>b3O;Rm^Ig*POZP8Rl0f zT~o=Nu1J|lO>}xX&#P58%Yl z83`HRs5#32Qm9mdCrMlV|NKNC+Z~ z9OB8xk5HJ>gBLi+m@(pvpw)1(OaVJKs*$Ou#@Knd#bk+V@y;YXT?)4eP9E5{J%KGtYinNYJUH9PU3A}66c>Xn zZ{Bn0<;8$WCOAL$^NqTjwM?5d=RHgw3!72WRo0c;+houoUA@HWLZM;^U$&sycWrFd zE7ekt9;kb0`lps{>R(}YnXlyGY}5pPd9zBpgXeJTY_jwaJGSJQC#-KJqmh-;ad&F- z-Y)E>!&`Rz!HtCz>%yOJ|v(u7P*I$jqEY3}(Z-orn4 zlI?CYKNl`6I){#2P1h)y(6?i;^z`N3bxTV%wNvQW+eu|x=kbj~s8rhCR*0H=iGkSj zk23lr9kr|p7#qKL=UjgO`@UnvzU)`&fI>1Qs7ubq{@+lK{hH* zvl6eSb9%yngRn^T<;jG1SVa)eA>T^XX=yUS@NCKpk?ovCW1D@!=@kn;l_BrG;hOTC z6K&H{<8K#dI(A+zw-MWxS+~{g$tI7|SfP$EYKxA}LlVO^sT#Oby^grkdZ^^lA}uEF zBSj$weBJG{+Bh@Yffzsw=HyChS(dtLE3i*}Zj@~!_T-Ay7z=B)+*~3|?w`Zd)Co2t zC&4DyB!o&YgSw+fJn6`sn$e)29`kUwAc+1MND7YjV%lO;H2}fNy>hD#=gT ze+-aFNpyKIoXY~Vq-}OWPBe?Rfu^{ps8>Xy%42r@RV#*QV~P83jdlFNgkPN=T|Kt7 zV*M`Rh*30&AWlb$;ae130e@}Tqi3zx2^JQHpM>j$6x`#{mu%tZlwx9Gj@Hc92IuY* zarmT|*d0E~vt6<+r?W^UW0&#U&)8B6+1+;k^2|FWBRP9?C4Rk)HAh&=AS8FS|NQaZ z2j!iZ)nbEyg4ZTp-zHwVlfLC~tXIrv(xrP8PAtR{*c;T24ycA-;auWsya-!kF~CWZ zw_uZ|%urXgUbc@x=L=_g@QJ@m#5beS@6W195Hn7>_}z@Xt{DIEA`A&V82bc^#!q8$ zFh?z_Vn|ozJ;NPd^5uu(9tspo8t%&-U9Ckay-s@DnM*R5rtu|4)~e)`z0P-sy?)kc zs_k&J@0&0!q4~%cKL)2l;N*T&0;mqX5T{Qy60%JtKTQZ-xb%KOcgqwJmb%MOOKk7N zgq})R_6**{8A|6H?fO+2`#QU)p$Ei2&nbj6TpLSIT^D$|`TcSeh+)}VMb}LmvZ{O| ze*1IdCt3+yhdYVxcM)Q_V0bIXLgr6~%JS<<&dxIgfL=Vnx4YHuU@I34JXA|+$_S3~ zy~X#gO_X!cSs^XM{yzDGNM>?v(+sF#<0;AH^YrE8smx<36bUsHbN#y57K8WEu(`qHvQ6cAZPo=J5C(lSmUCZ57Rj6cx!e^rfaI5%w}unz}4 zoX=nt)FVNV%QDJH`o!u9olLD4O5fl)xp+#RloZlaA92o3x4->?rB4`gS$;WO{R;Z3>cG3IgFX2EA?PK^M}@%1%A;?f6}s&CV$cIyEr#q5;yHdNZ9h{| z-=dX+a5elJoDo?Eq&Og!nN6A)5yYpnGEp}?=!C-V)(*~z-+?kY1Q7qs#Rsy%hu_60rdbB+QQNr?S1 z?;xtjUv|*E3}HmuNyB9aFL5H~3Ho0UsmuMZELp1a#CA1g`P{-mT?BchuLEtK}!QZ=3AWakRu~?f9V~3F;TV`5%9Pcs_$gq&CcU}r8gOO zC2&SWPsSG{&o-LIGTBqp6SLQZPvYKp$$7L4WRRZ0BR$Kf0I0SCFkqveCp@f)o8W)! z$%7D1R`&j7W9Q9CGus_)b%+B#J2G;l*FLz#s$hw{BHS~WNLODV#(!u_2Pe&tMsq={ zdm7>_WecWF#D=?eMjLj=-_z`aHMZ=3_-&E8;ibPmM}61i6J3is*=dKf%HC>=xbj4$ zS|Q-hWQ8T5mWde6h@;mS+?k=89?1FU<%qH9B(l&O>k|u_aD|DY*@~(`_pb|B#rJ&g zR0(~(68fpUPz6TdS@4JT5MOPrqDh5_H(eX1$P2SQrkvN8sTxwV>l0)Qq z0pzTuvtEAKRDkKGhhv^jk%|HQ1DdF%5oKq5BS>szk-CIke{%js?~%@$uaN3^Uz6Wf z_iyx{bZ(;9y4X&>LPV=L=d+A}7I4GkK0c1Xts{rrW1Q7apHf-))`BgC^0^F(>At1* za@e7{lq%yAkn*NH8Q1{@{lKhRg*^TfGvv!Sn*ed*x@6>M%aaqySxR|oNadYt1mpUZ z6H(rupHYf&Z z29$5g#|0MX#aR6TZ$@eGxxABRKakDYtD%5BmKp;HbG_ZbT+=81E&=XRk6m_3t9PvD zr5Cqy(v?gHcYvYvXkNH@S#Po~q(_7MOuCAB8G$a9BC##gw^5mW16cML=T=ERL7wsk zzNEayTG?mtB=x*wc@ifBCJ|irFVMOvH)AFRW8WE~U()QT=HBCe@s$dA9O!@`zAAT) zaOZ7l6vyR+Nk_OOF!ZlZmjoImKh)dxFbbR~z(cMhfeX1l7S_`;h|v3gI}n9$sSQ>+3@AFAy9=B_y$)q;Wdl|C-X|VV3w8 z2S#>|5dGA8^9%Bu&fhmVRrTX>Z7{~3V&0UpJNEl0=N32euvDGCJ>#6dUSi&PxFW*s zS`}TB>?}H(T2lxBJ!V#2taV;q%zd6fOr=SGHpoSG*4PDaiG0pdb5`jelVipkEk%FV zThLc@Hc_AL1#D&T4D=w@UezYNJ%0=f3iVRuVL5H?eeZM}4W*bomebEU@e2d`M<~uW zf#Bugwf`VezG|^Qbt6R_=U0}|=k;mIIakz99*>FrsQR{0aQRP6ko?5<7bkDN8evZ& zB@_KqQG?ErKL=1*ZM9_5?Pq%lcS4uLSzN(Mr5=t6xHLS~Ym`UgM@D&VNu8e?_=nSFtF$u@hpPSmI4Vo_t&v?>$~K4y(O~Rb*(MFy_igM7 z*~yYUyR6yQgzWnWMUgDov!!g=lInM+=lOmOk4L`O?{i&qxy&D*_qorRbDwj6?)!ef z#JLd7F6Z2I$S0iYI={rZNk*<{HtIl^mx=h>Cim*04K4+Z4IJtd*-)%6XV2(MCscPiw_a+y*?BKbTS@BZ3AUao^%Zi#PhoY9Vib4N>SE%4>=Jco0v zH_Miey{E;FkdlZSq)e<{`+S3W=*ttvD#hB8w=|2aV*D=yOV}(&p%0LbEWH$&@$X3x~CiF-?ejQ*N+-M zc8zT@3iwkdRT2t(XS`d7`tJQAjRmKAhiw{WOqpuvFp`i@Q@!KMhwKgsA}%@sw8Xo5Y=F zhRJZg)O4uqNWj?V&&vth*H#je6T}}p_<>!Dr#89q@uSjWv~JuW(>FqoJ5^ho0%K?E z9?x_Q;kmcsQ@5=}z@tdljMSt9-Z3xn$k)kEjK|qXS>EfuDmu(Z8|(W?gY6-l z@R_#M8=vxKMAoi&PwnaIYw2COJM@atcgfr=zK1bvjW?9B`-+Voe$Q+H$j!1$Tjn+* z&LY<%)L@;zhnJlB^Og6I&BOR-m?{IW;tyYC%FZ!&Z>kGjHJ6cqM-F z&19n+e1=9AH1VrVeHrIzqlC`w9=*zfmrerF?JMzO&|Mmv;!4DKc(sp+jy^Dx?(8>1 zH&yS_4yL7m&GWX~mdfgH*AB4{CKo;+egw=PrvkTaoBU+P-4u?E|&!c z)DKc;>$$B6u*Zr1SjUh2)FeuWLWHl5TH(UHWkf zLs>7px!c5n;rbe^lO@qlYLzlDVp(z?6rPZel=YB)Uv&n!2{+Mb$-vQl=xKw( zve&>xYx+jW_NJh!FV||r?;hdP*jOXYcLCp>DOtJ?2S^)DkM{{Eb zS$!L$e_o0(^}n3tA1R3-$SNvgBq;DOEo}fNc|tB%%#g4RA3{|euq)p+xd3I8^4E&m zFrD%}nvG^HUAIKe9_{tXB;tl|G<%>yk6R;8L2)KUJw4yHJXUOPM>(-+jxq4R;z8H#>rnJy*)8N+$wA$^F zN+H*3t)eFEgxLw+Nw3};4WV$qj&_D`%ADV2%r zJCPCo%{=z7;`F98(us5JnT(G@sKTZ^;2FVitXyLe-S5(hV&Ium+1pIUB(CZ#h|g)u zSLJJ<@HgrDiA-}V_6B^x1>c9B6%~847JkQ!^KLZ2skm;q*edo;UA)~?SghG8;QbHh z_6M;ouo_1rq9=x$<`Y@EA{C%6-pEV}B(1#sDoe_e1s3^Y>n#1Sw;N|}8D|s|VPd+g z-_$QhCz`vLxxrVMx3ape1xu3*wjx=yKSlM~nFgkNWb4?DDr*!?U)L_VeffF<+!j|b zZ$Wn2$TDv3C3V@BHpSgv3JUif8%hk%OsGZ=OxH@8&4`bbf$`aAMchl^qN>Eyu3JH} z9-S!x8-s4fE=lad%Pkp8hAs~u?|uRnL48O|;*DEU! zuS0{cpk%1E0nc__2%;apFsTm0bKtd&A0~S3Cj^?72-*Owk3V!ZG*PswDfS~}2<8le z5+W^`Y(&R)yVF*tU_s!XMcJS`;(Tr`J0%>p=Z&InR%D3@KEzzI+-2)HK zuoNZ&o=wUC&+*?ofPb0a(E6(<2Amd6%uSu_^-<1?hsxs~0K5^f(LsGqgEF^+0_H=uNk9S0bb!|O8d?m5gQjUKevPaO+*VfSn^2892K~%crWM8+6 z25@V?Y@J<9w%@NXh-2!}SK_(X)O4AM1-WTg>sj1{lj5@=q&dxE^9xng1_z9w9DK>| z6Iybcd0e zyi;Ew!KBRIfGPGytQ6}z}MeXCfLY0?9%RiyagSp_D1?N&c{ zyo>VbJ4Gy`@Fv+5cKgUgs~na$>BV{*em7PU3%lloy_aEovR+J7TfQKh8BJXyL6|P8un-Jnq(ghd!_HEOh$zlv2$~y3krgeH;9zC}V3f`uDtW(%mT#944DQa~^8ZI+zAUu4U(j0YcDfKR$bK#gvn_{JZ>|gZ5+)u?T$w7Q%F^;!Wk?G z(le7r!ufT*cxS}PR6hIVtXa)i`d$-_1KkyBU>qmgz-=T};uxx&sKgv48akIWQ89F{ z0XiY?WM^~;|T8zBOr zs#zuOONzH?svv*jokd5SK8wG>+yMC)LYL|vLqm^PMHcT=`}V$=nIRHe2?h)8WQa6O zPAU}d`1y(>kZiP~Gr=mtJLMu`i<2CspL|q2DqAgAD^7*$xzM`PU4^ga`ilE134XBQ z99P(LhHU@7qvl9Yzg$M`+dlS=x^(m-_3t|h>S}E0bcFMn=C|KamQ)=w2^e)35p`zY zRV8X?d;s^>Cof2SPR&nP3E+-LCkS0J$H!eh8~k0qo$}00b=7!H_I2O+Ro@3O$nPdm ztmbOO^B+IHzQ5w>@@@J4cKw5&^_w6s!s=H%&byAbUtczPQ7}wfTqxxtQNfn*u73Qw zGuWsrky_ajPx-5`R<)6xHf>C(oqGf_Fw|-U*GfS?xLML$kv;h_pZ@Kk$y0X(S+K80 z6^|z)*`5VUkawg}=z`S;VhZhxyDfrE0$(PMurAxl~<>lfZa>JZ288ULK7D` zl9|#L^JL}Y$j*j`0-K6kH#?bRmg#5L3iB4Z)%iF@SqT+Lp|{i`m%R-|ZE94Np7Pa5 zCqC^V3}B(FR340pmF*qaa}M}+h6}mqE~7Sh!9bDv9YRT|>vBNAqv09zXHMlcuhKD| zcjjA(b*XCIwJ33?CB!+;{)vX@9xns_b-VO{i0y?}{!sdXj1GM8+$#v>W7nw;+O_9B z_{4L;C6ol?(?W0<6taGEn1^uG=?Q3i29sE`RfYCaV$3DKc_;?HsL?D_fSYg}SuO5U zOB_f4^vZ_x%o`5|C@9C5+o=mFy@au{s)sKw!UgC&L35aH(sgDxRE2De%(%OT=VUdN ziVLEmdOvJ&5*tCMKRyXctCwQu_RH%;m*$YK&m;jtbdH#Ak~13T1^f89tn`A%QEHWs~jnY~E}p_Z$XC z=?YXLCkzVSK+Id`xZYTegb@W8_baLt-Fq`Tv|=)JPbFsKRm)4UW;yT+J`<)%#ue9DPOkje)YF2fsCilK9MIIK>p*`fkoD5nGfmLwt)!KOT+> zOFq*VZktDDyM3P5UOg`~XL#cbzC}eL%qMB=Q5$d89MKuN#$6|4gx_Jt0Gfn8w&q}%lq4QU%6#jT*MRT% zrLz~C8FYKHawn-EQWN1B75O&quS+Z81(zN)G>~vN8VwC+e+y(`>HcxC{MrJ;H1Z4k zZWuv$w_F0-Ub%MVcpIc){4PGL^I7M{>;hS?;eH!;gmcOE66z3;Z1Phqo(t zVP(Hg6q#0gIKgsg7L7WE!{Y#1nI(45tx2{$34dDd#!Z0NIyrm)HOn5W#7;f4pQci# zDW!FI(g4e668kI9{2+mLwB+=#9bfqgX%!B34V-$wwSN(_cm*^{y0jQtv*4}eO^sOV z*9xoNvX)c9isB}Tgx&ZRjp3kwhTVK?r9;n!x>^XYT z@Q^7zp{rkIs{2mUSE^2!Gf6$6;j~&4=-0cSJJDizZp6LTe8b45;{AKM%v99}{{FfC zz709%u0mC=1KXTo(=TqmZQ;c?$M3z(!xah>aywrj40sc2y3rKFw4jCq+Y+u=CH@_V zxz|qeTwa>+<|H%8Dz5u>ZI5MmjTFwXS-Fv!TDd*`>3{krWoNVx$<133`(ftS?ZPyY z&4@ah^3^i`vL$BZa>O|Nt?ucewzsF)0zX3qmM^|waXr=T0pfIb0*$AwU=?Ipl|1Y; z*Pk6{C-p4MY;j@IJ|DW>QHZQJcp;Z~?8(Q+Kk3^0qJ}SCk^*n4W zu9ZFwLHUx-$6xvaQ)SUQcYd6fF8&x)V`1bIuX@>{mE$b|Yd(qomn3;bPwnDUc0F=; zh*6_((%bqAYQWQ~odER?h>1mkL4kpb3s7`0m@rDKGU*oyF)$j~Ffd4fXV$?`f~rHf zB%Y)@5SXZvfwm10RY5X?TEo)PK_`L6qgBp=#>fO49$D zDq8Ozj0q6213tV5Qq=;fZ0$|KroY{Dz=l@lU^J)?Ko@ti20TRplXzphBi>XGx4bou zEWrkNjz0t5j!_ke{g5I#PUlEU$Km8g8TE|XK=MkU@PT4T><2OVamoK;wJ}3X0L$vX zgd7gNa359*nc)R-0!`2X@FOTB`+oETOPc=ubp5R)VQgY+5BTZZJ2?9QwnO=dnulIUF3gFn;BODC2)65)HeVd%t86sL7Rv^Y+nbn+&l z6BAJY(ETvwI)Ts$aiE8rht4KD*qNyE{8{x6R|%akbTBzw;2+6Echkt+W+`u^XX z_z&x%n5IFJ|DjL!e)upfGNX(kojugZ3I`oH1PvW`wFW_ske0j@lB9bX zO;2)`y+|!@X(fZ1<2n!Qx*)_^Ai@Cv-dF&(vnudG?0CsddG_&Wtae(n|K59ew)6St z#dj7_(Cfwzh$H$5M!$UDd8=4>IQsD3xV=lXUq($;(h*$0^yd+b{qq63f0r_de#!o_ zXDngc>zy`uor)4A^2M#U*DC~i+dc<)Tb1Tv&~Ev@oM)5iJ4Sn#8iRw16XXuV50BS7 zdBL5Mefch(&^{luE{*5qtCZk$oFr3RH=H!c3wGR=HJ(yKc_re_X9pD` zJ;uxPzUfVpgU>DSq?J;I@a+10l0ONXPcDkiYcihREt5~T5Gb}sT0+6Q;AWHl`S5dV>lv%-p9l#xNNy7ZCr%cyqHY%TZ8Q4 zbp&#ov1*$#grNG#1vgfFOLJCaNG@K|2!W&HSh@3@Y%T?3YI75bJp!VP*$*!< z;(ffNS_;@RJ`=c7yX04!u3JP*<8jeqLHVJu#WV&v6wA!OYJS4h<_}^QI&97-;=ojW zQ-1t)7wnxG*5I%U4)9$wlv5Fr;cIizft@&N+32O%B{R1POm$oap@&f| zh+5J{>U6ftv|vAeKGc|zC=kO(+l7_cLpV}-D#oUltScw})N>~JOZLU_0{Ka2e1evz z{^a*ZrLr+JUj;)K&u2CoCAXLC2=fVScI(m_p~0FmF>>&3DHziouln?;sxW`NB}cSX z8?IsJB)Z=aYRz!X=yJn$kyOWK%rCYf-YarNqKzmWu$ZvkP12b4qH zhS9Q>j<}(*frr?z<%9hl*i^#@*O2q(Z^CN)c2c z>1B~D;@YpG?G!Yk+*yn4vM4sO-_!&m6+`k|3zd;8DJnxsBYtI;W3We+FN@|tQ5EW= z!VU>jtim0Mw#iaT8t_<+qKIEB-WwE04lBd%Letbml9N!?SLrEG$nmn7&W(W`VB@5S zaY=sEw2}i@F_1P4OtEw?xj4@D6>_e=m=797#hg}f*l^`AB|Y0# z9=)o|%TZFCY$SzgSjS|8AI-%J4x}J)!IMxY3_KYze`_I=c1nmrk@E8c9?MVRu)7+Ue79|)rBX7tVB7U|w4*h(;Gi3D9le49B38`wuv zp7{4X^p+K4*$@gU(Tq3K1a#3SmYhvI42)GzG4f|u zwQFT1n_=n|jpi=70-yE9LA+d*T8u z`=VmmXJ_f6WmZveZPct$Cgu^~gFiyL>Lnpj*6ee>*0pz=t$IJ}+rE zsf@>jlcG%Wx;Cp5x)YSVvB1$yyY1l&o zvwX=D7k)Dn;ciX?Z)Pn8$flC8#m`nB&(8?RSdBvr?>T9?E$U3uIX7T?$v4dWCa46 z+&`ot8ZTEgp7G+c52oHJ8nw5}a^dwb_l%MOh(ebVj9>_koQP^$2B~eUfSbw9RY$_< z&DDWf2LW;b0ZDOaZ&2^i^g+5uTd;GwO(-bbo|P^;CNL-%?9mRmxEw~5&z=X^Rvbo^WJW=n_%*7974RY}JhFv46> zd}`2|qkd;89l}R;i~9T)V-Q%K)O=yfVKNM4Gbacc7AOd>#^&W&)Xx!Uy5!BHnp9kh z`a(7MO6+Ren#>R^D0K)1sE{Bv>}s6Rb9MT14u!(NpZOe-?4V=>qZ>}uS)!y~;jEUK z&!U7Fj&{WdgU#L0%bM}SYXRtM5z!6M+kgaMKt%3FkjWYh=#QUpt$XX1!*XkpSq-pl zhMe{muh#knk{9_V3%qdDcWDv}v)m4t9 zQhv{;} zc{}#V^N3H>9mFM8`i`0p+fN@GqX+kl|M94$BK3J-X`Hyj8r!#x6Vt(PXjn?N)qedP z=o1T^#?1^a{;bZ&x`U{f?}TMo8ToN zkHj5v|}r}wDEi7I@)Gj+S1aE-GdnLN+$hw!=DzglMaj#{qjXi_dwpr|HL(gcCXwGLEmi|{4&4#OZ4ChceA zKVd4K!D>_N=_X;{poT~4Q+!Le+ZV>=H7v1*l%w`|`Dx8{)McN@NDlQyln&N3@bFpV z_1w~O4EH3fF@IzJ9kDk@7@QctFq8FbkbaH7K$iX=bV~o#gfh?2JD6lZf(XP>~DACF)fGFt)X%-h1yY~MJU{nA5 ze2zxWMs{YdX3q5XU*9hOH0!_S24DOBA5usB+Ws$6{|AMe*joJ?RxfV}*7AKN9V*~J zK+OMcE@bTD>TG1*yc?*qGqjBN8mgg@h1cJLDv)0!WRPIkC` zZrWXrceVw;fB%3`6kq=a!pq|hFIsQ%ZSlo~)D z|64!aCnw-?>}AG|*iOl44KVf8@|joXi&|)1rB;EQWgm+iHfVbgllP$f!$Wf42%NO5b(j9Bw6L z;0dpUUK$5GX4QbMlTmLM_jJt!ur`_0~$b#BB7FL*%XFf<b__1o)Ao3rlobbN8-(T!1d-bR8D3S0@d zLI!*GMb5s~Q<&sjd}lBb8Nr0>PqE6_!3!2d(KAWFxa{hm`@u|a(%#i(#f8{BP2wbs zt+N_slWF4IF_O|{w`c~)Xvh&R{Au~CFmW#0+}MBd2~X}t9lz6*E7uAD`@EBDe$>7W zzPUkJx<`f$0VA$=>R57^(K^h86>09?>_@M(R4q($!Ck6GG@pnu-x*exAx1jOv|>KH zjNfG5pwm`E-=ydcb+3BJwuU;V&OS=6yM^4Jq{%AVqnTTLwV`AorIDD}T&jWr8pB&j28fVtk_y*JRP^t@l*($UZ z6(B^-PBNZ+z!p?+e8@$&jCv^EWLb$WO=}Scr$6SM*&~B95El~;W_0(Bvoha|uQ1T< zO$%_oLAwf1bW*rKWmlD+@CP&$ObiDy=nh1b2ejz%LO9937N{LDe7gle4i!{}I$;&Y zkexJ9Ybr+lrCmKWg&}p=`2&Gf10orS?4$VrzWidT=*6{KzOGMo?KI0>GL0{iFWc;C z+LPq%VH5g}6V@-tg2m{C!-$fapJ9y}c$U}aUmS{9#0CM*8pC|sfer!)nG7Ji>mfRh z+~6CxNb>6eWKMHBz-w2{mLLwdA7dA-qfTu^A2yG1+9s5k zcF=le_UPYG&q!t5Zd_*E_P3Cf5T6821bO`daa`;DODm8Ih8k89=RN;-asHIigj`n=ux>*f!OC5#;X5i;Q z+V!GUy0|&Y_*8k_QRUA8$lHP;GJ3UUD08P|ALknng|YY13)}!!HW@0z$q+kCH%xet zlWf@BXQ=b=4}QO5eNnN~CzWBbHGUivG=`&eWK}beuV*;?zt=P#pM*eTuy3 zP}c#}AXJ0OIaqXji78l;YrP4sQe#^pOqwZUiiN6^0RCd#D271XCbEKpk`HI0IsN^s zES7YtU#7=8gTn#lkrc~6)R9u&SX6*Jk4GFX7){E)WE?pT8a-%6P+zS6o&A#ml{$WX zABFz#i7`DDlo{34)oo?bOa4Z_lNH>n;f0nbt$JfAl~;4QY@}NH!X|A$KgMmEsd^&Y zt;pi=>AID7ROQfr;MsMtClr5b0)xo|fwhc=qk33wQ|}$@?{}qXcmECh>#kUQ-If0$ zseb{Wf4VFGLNc*Rax#P8ko*=`MwaR-DQ8L8V8r=2N{Gaips2_^cS|oC$+yScRo*uF zUO|5=?Q?{p$inDpx*t#Xyo6=s?bbN}y>NNVxj9NZCdtwRI70jxvm3!5R7yiWjREEd zDUjrsZhS|P&|Ng5r+f^kA6BNN#|Se}_GF>P6sy^e8kBrgMv3#vk%m}9PCwUWJg-AD zFnZ=}lbi*mN-AOm zCs)r=*YQAA!`e#1N>aHF=bb*z*hXH#Wl$z^o}x##ZrUc=kh%OHWhp=7;?8%Xj||@V?1c ziWoaC$^&04;A|T)!Zd9sUzE&$ODyJaBpvqsw19Uiuq{i#VK1!htkdRWBnb z`{rat=nHArT%^R>u#CjjCkw-7%g53|&7z-;X+ewb?OLWiV|#nuc8mp*LuGSi3IP<<*Wyo9GKV7l0Noa4Jr0g3p_$ z*R9{qn=?IXC#WU>48-k5V2Oc_>P;4_)J@bo1|pf=%Rcbgk=5m)CJZ`caHBTm3%!Z9 z_?7LHr_BXbKKr=JD!%?KhwdYSdu8XxPoA{n8^%_lh5cjRHuCY9Zlpz8g+$f@bw@0V z+6DRMT9c|>1^3D|$Vzc(C?M~iZurGH2pXPT%F!JSaAMdO%!5o0uc&iqHx?ImcX6fI zCApkzc~OOnfzAd_+-DcMp&AOQxE_EsMqKM{%dRMI5`5CT&%mQO?-@F6tE*xL?aEGZ z8^wH@wRl`Izx4sDmU>}Ym{ybUm@F83qqZPD6nFm?t?(7>h*?`fw)L3t*l%*iw0Qu#?$5eq!Qc zpQvqgSxrd83NsdO@lL6#{%lsYXWen~d3p4fGBb7&5xqNYJ)yn84!e1PmPo7ChVd%4 zHUsV0Mh?VpzZD=A6%)Qrd~i7 z96*RPbid;BN{Wh?adeD_p8YU``kOrGkNox3D9~!K?w>#kFz!4lzOWR}puS(DmfjJD z`x0z|qB33*^0mZdM&6$|+T>fq>M%yoy(BEjuh9L0>{P&XJ3enGpoQRx`v6$txXt#c z0#N?b5%srj(4xmPvJxrlF3H%OMB!jvfy z;wx8RzU~lb?h_}@V=bh6p8PSb-dG|-T#A?`c&H2`_!u+uenIZe`6f~A7r)`9m8atC zt(b|6Eg#!Q*DfRU=Ix`#B_dK)nnJ_+>Q<1d7W)eynaVn`FNuN~%B;uO2}vXr5^zi2 z!ifIF5@Zlo0^h~8+ixFBGqtweFc`C~JkSq}&*a3C}L?b5Mh-bW=e)({F_g4O3 zb@SFTK3VD9QuFgFnK4Ve_pXc3{S$=+Z;;4+;*{H}Rc;845rP?DLK6G5Y-xdUKkA6E3Dz&5f{F^FjJQ(NSpZ8q-_!L3LL@H* zxbDF{gd^U3uD;)a)sJwAVi}7@%pRM&?5IaUH%+m{E)DlA_$IA1=&jr{KrhD5q&lTC zAa3c)A(K!{#nOvenH6XrR-y>*4M#DpTTOGQEO5Jr6kni9pDW`rvY*fs|ItV;CVITh z=`rxcH2nEJpkQ^(;1c^hfb8vGN;{{oR=qNyKtR1;J>CByul*+=`NydWnSWJR#I2lN zTvgnR|MBx*XFsfdA&;tr^dYaqRZp*2NwkAZE6kV@1f{76e56eUmGrZ>MDId)oqSWw z7d&r3qfazg+W2?bT}F)4jD6sWaw`_fXZGY&wnGm$FRPFL$HzVTH^MYBHWGCOk-89y zA+n+Q6EVSSCpgC~%uHfvyg@ufE^#u?JH?<73A}jj5iILz4Qqk5$+^U(SX(-qv5agK znUkfpke(KDn~dU0>gdKqjTkVk`0`9^0n_wzXO7R!0Thd@S;U`y)VVP&mOd-2 z(hT(|$=>4FY;CBY9#_lB$;|Wd$aOMT5O_3}DYXEHn&Jrc3`2JiB`b6X@EUOD zVl0S{ijm65@n^19T3l%>*;F(?3r3s?zY{thc4%AD30CeL_4{8x6&cN}zN3fE+x<9; zt2j1RRVy5j22-8U8a6$pyT+<`f+x2l$fd_{qEp_bfxfzu>ORJsXaJn4>U6oNJ#|~p z`*ZC&NPXl&=vq2{Ne79AkQncuxvbOG+28*2wU$R=GOmns3W@HE%^r)Fu%Utj=r9t` zd;SVOnA(=MXgnOzI2@3SGKHz8HN~Vpx&!Ea+Df~`*n@8O=0!b4m?7cE^K*~@fqv9q zF*uk#1@6Re_<^9eElgJD!nTA@K9C732tV~;B`hzZ321Ph=^BH?zXddiu{Du5*IPg} zqDM=QxjT!Rp|#Bkp$(mL)aar)f(dOAXUiw81pX0DC|Y4;>Vz>>DMshoips^8Frdv} zlTD=cKa48M>dR<>(YlLPOW%rokJZNF2gp8fwc8b2sN+i6&-pHr?$rj|uFgktK@jg~ zIFS(%=r|QJ=$kvm_~@n=ai1lA{7Z}i+zj&yzY+!t$iGUy|9jH#&oTNJ;JW-3n>DF+ z3aCOzqn|$X-Olu_p7brzn`uk1F*N4@=b=m;S_C?#hy{&NE#3HkATrg?enaVGT^$qIjvgc61y!T$9<1B@?_ibtDZ{G zeXInVr5?OD_nS_O|CK3|RzzMmu+8!#Zb8Ik;rkIAR%6?$pN@d<0dKD2c@k2quB%s( zQL^<_EM6ow8F6^wJN1QcPOm|ehA+dP(!>IX=Euz5qqIq}Y3;ibQtJnkDmZ8c8=Cf3 zu`mJ!Q6wI7EblC5RvP*@)j?}W=WxwCvF3*5Up_`3*a~z$`wHwCy)2risye=1mSp%p zu+tD6NAK3o@)4VBsM!@);qgsjgB$kkCZhaimHg&+k69~drbvRTacWKH;YCK(!rC?8 zP#cK5JPHSw;V;{Yji=55X~S+)%(8fuz}O>*F3)hR;STU`z6T1aM#Wd+FP(M5*@T1P z^06O;I20Sk!bxW<-O;E081KRdHZrtsGJflFRRFS zdi5w9OVDGSL3 zNrC7GVsGN=b;YH9jp8Z2$^!K@h=r-xV(aEH@#JicPy;A0k1>g1g^XeR`YV2HfmqXY zYbRwaxHvf}OlCAwHoVI&QBLr5R|THf?nAevV-=~V8;gCsX>jndvNOcFA+DI+zbh~# zZ7`qNk&w+_+Yp!}j;OYxIfx_{f0-ONc?mHCiCUak=>j>~>YR4#w# zuKz~UhT!L~GfW^CPqG8Lg)&Rc6y^{%3H7iLa%^l}cw_8UuG;8nn9)kbPGXS}p3!L_ zd#9~5CrH8xtUd?{d2y^PJg+z(xIfRU;`}^=OlehGN2=?}9yH$4Rag}*+AWotyxfCJ zHx=r7ZH>j2kV?%7WTtp+-HMa0)_*DBBmC{sd$)np&GEJ__kEd`xB5a2A z*J+yx>4o#ZxwA{;NjhU*1KT~=ZK~GAA;KZHDyBNTaWQ1+;tOFFthnD)DrCn`DjBZ% zk$N5B4^$`n^jNSOr=t(zi8TN4fpaccsb`zOPD~iY=UEK$0Y70bG{idLx@IL)7^(pL z{??Bnu=lDeguDrd%qW1)H)H`9otsOL-f4bSu};o9OXybo6J!Lek`a4ff>*O)BDT_g z<6@SrI|C9klY(>_PfA^qai7A_)VNE4c^ZjFcE$Isp>`e5fLc)rg@8Q_d^Uk24$2bn z9#}6kZ2ZxS9sI(RqT7?El2@B+($>eBQrNi_k#CDJ8D9}8$mmm z4oSKO^F$i+NG)-HE$O6s1--6EzJa?C{x=QgK&c=)b(Q9OVoAXYEEH20G|q$}Hue%~ zO3B^bF=t7t48sN zWh_zA`w~|){-!^g?6Mqf6ieV zFx~aPUOJGR=4{KsW7I?<=J2|lY`NTU=lt=%JE9H1vBpkcn=uq(q~=?iBt_-r(PLBM zP-0dxljJO>4Wq-;stY)CLB4q`-r*T$!K2o}?E-w_i>3_aEbA^MB7P5piwt1dI-6o!qWCy0 ztYy!x9arGTS?kabkkyv*yxvsPQ7Vx)twkS6z2T@kZ|kb8yjm+^$|sEBmvACeqbz)RmxkkDQX-A*K!YFziuhwb|ym>C$}U|J)4y z$(z#)GH%uV6{ec%Zy~AhK|+GtG8u@c884Nq%w`O^wv2#A(&xH@c5M`Vjk*SR_tJnq z0trB#aY)!EKW_}{#L3lph5ow=@|D5LzJYUFD6 z7XnUeo_V0DVSIKMFD_T0AqAO|#VFDc7c?c-Q%#u00F%!_TW1@JVnsfvm@_9HKWflBOUD~)RL``-!P;(bCON_4eVdduMO>?IrQ__*zE@7(OX zUtfH@AX*53&xJW*Pu9zcqxGiM>xol0I~QL5B%Toog3Jlenc^WbVgeBvV8C8AX^Vj& z^I}H})B=VboO%q1;aU5ACMh{yK4J;xlMc`jCnZR^!~LDs_MP&8;dd@4LDWw~*>#OT zeZHwdQWS!tt5MJQI~cw|Ka^b4c|qyd_ly(+Ql2m&AAw^ zQeSXDOOH!!mAgzAp0z)DD>6Xo``b6QwzUV@w%h}Yo>)a|xRi$jGuHQhJVA%>)PUvK zBQ!l0hq<3VZ*RnrDODP)>&iS^wf64C;MGqDvx>|p;35%6(u+IHoNbK z;Gb;TneFo*`zUKS6kwF*&b!U8e5m4YAo03a_e^!5BP42+r)LFhEy?_7U1IR<; z^0v|DhCYMSj<-;MtY%R@Fg;9Kky^pz_t2nJfKWfh5Eu@_l{^ph%1z{jkg5jQrkvD< z#vdK!nku*RrH~TdN~`wDs;d>XY1PH?O<4^U4lmA|wUW{Crrv#r%N>7k#{Gc44Fr|t z@UZP}Y-TrAmnEZ39A*@6;ccsR>)$A)S>$-Cj!=x$rz7IvjHIPM(TB+JFf{ehuIvY$ zsDAwREg*%|=>Hw$`us~RP&3{QJg%}RjJKS^mC_!U;E5u>`X`jW$}P`Mf}?7G7FX#{ zE(9u1SO;3q@ZhDL9O({-RD+SqqPX)`0l5IQu4q)49TUTkxR(czeT}4`WV~pV*KY&i zAl3~X%D2cPVD^B43*~&f%+Op)wl<&|D{;=SZwImydWL6@_RJjxP2g)s=dH)u9Npki zs~z9A+3fj0l?yu4N0^4aC5x)Osnm0qrhz@?nwG_`h(71P znbIewljU%T*cC=~NJy|)#hT+lx#^5MuDDnkaMb*Efw9eThXo|*WOQzJ*#3dmRWm@! zfuSc@#kY{Um^gBc^_Xdxnl!n&y&}R4yAbK&RMc+P^Ti;YIUh|C+K1|=Z^{nZ}}rxH*v{xR!i%qO~o zTr`WDE@k$M9o0r4YUFFeQO7xCu_Zgy)==;fCJ94M_rLAv&~NhfvcLWCoaGg2ao~3e zBG?Ms9B+efMkp}7BhmISGWmJsKI@a8b}4lLI48oWKY|8?zuuNc$lt5Npr+p7a#sWu zh!@2nnLBVJK!$S~>r2-pN||^w|fY`CT{TFnJy`B|e5;=+_v4l8O-fkN&UQbA4NKTyntd zqK{xEKh}U{NHoQUf!M=2(&w+eef77VtYr;xs%^cPfKLObyOV_9q<(%76-J%vR>w9!us-0c-~Y?_EVS%v!* z15s2s3eTs$Osz$JayyH|5nPAIPEX=U;r&p;K14G<1)bvn@?bM5kC{am|C5%hyxv}a z(DeSKI5ZfZ1*%dl8frIX2?);R^^~LuDOpNpk-2R8U1w92HmG1m&|j&J{EK=|p$;f9 z7Rs5|jr4r8k5El&qcuM+YRlKny%t+1CgqEWO>3;BSRZi(LA3U%Jm{@{y+A+w(gzA< z7dBq6a1sEWa4cD0W7=Ld9z0H7RI^Z7vl(bfA;72j?SWCo`#5mVC$l1Q2--%V)-uN* z9ha*s-AdfbDZ8R8*fpwjzx=WvOtmSzGFjC#X)hD%Caeo^OWjS(3h|d9_*U)l%{Ab8 zfv$yoP{OuUl@$(-sEVNt{*=qi5P=lpxWVuz2?I7Dc%BRc+NGNw+323^ z5BXGfS71oP^%apUo(Y#xkxE)y?>BFzEBZ}UBbr~R4$%b7h3iZu3S(|A;&HqBR{nK& z$;GApNnz=kNO^FL&nYcfpB7Qg;hGJPsCW44CbkG1@l9pn0`~oKy5S777uH)l{irK!ru|X+;4&0D;VE*Ii|<3P zUx#xUqvZT5kVQxsF#~MwKnv7;1pR^0;PW@$@T7I?s`_rD1EGUdSA5Q(C<>5SzE!vw z;{L&kKFM-MO>hy#-8z`sdVx})^(Dc-dw;k-h*9O2_YZw}|9^y-|8RQ`BWJUJL(Cer zP5Z@fNc>pTXABbTRY-B5*MphpZv6#i802giwV&SkFCR zGMETyUm(KJbh+&$8X*RB#+{surjr;8^REEt`2&Dubw3$mx>|~B5IKZJ`s_6fw zKAZx9&PwBqW1Oz0r0A4GtnZd7XTKViX2%kPfv+^X3|_}RrQ2e3l=KG_VyY`H?I5&CS+lAX5HbA%TD9u6&s#v!G> zzW9n4J%d5ye7x0y`*{KZvqyXUfMEE^ZIffzI=Hh|3J}^yx7eL=s+TPH(Q2GT-sJ~3 zI463C{(ag7-hS1ETtU;_&+49ABt5!A7CwLwe z=SoA8mYZIQeU;9txI=zcQVbuO%q@E)JI+6Q!3lMc=Gbj(ASg-{V27u>z2e8n;Nc*pf}AqKz1D>p9G#QA+7mqqrEjGfw+85Uyh!=tTFTv3|O z+)-kFe_8FF_EkTw!YzwK^Hi^_dV5x-Ob*UWmD-})qKj9@aE8g240nUh=g|j28^?v7 zHRTBo{0KGaWBbyX2+lx$wgXW{3aUab6Bhm1G1{jTC7ota*JM6t+qy)c5<@ zpc&(jVdTJf(q3xB=JotgF$X>cxh7k*(T`-V~AR+`%e?YOeALQ2Qud( zz35YizXt(aW3qndR}fTw1p()Ol4t!D1pitGNL95{SX4ywzh0SF;=!wf=?Q?_h6!f* zh7<+GFi)q|XBsvXZ^qVCY$LUa{5?!CgwY?EG;*)0ceFe&=A;!~o`ae}Z+6me#^sv- z1F6=WNd6>M(~ z+092z>?Clrcp)lYNQl9jN-JF6n&Y0mp7|I0dpPx+4*RRK+VQI~>en0Dc;Zfl+x z_e_b7s`t1_A`RP3$H}y7F9_na%D7EM+**G_Z0l_nwE+&d_kc35n$Fxkd4r=ltRZhh zr9zER8>j(EdV&Jgh(+i}ltESBK62m0nGH6tCBr90!4)-`HeBmz54p~QP#dsu%nb~W z7sS|(Iydi>C@6ZM(Us!jyIiszMkd)^u<1D+R@~O>HqZIW&kearPWmT>63%_t2B{_G zX{&a(gOYJx!Hq=!T$RZ&<8LDnxsmx9+TBL0gTk$|vz9O5GkK_Yx+55^R=2g!K}NJ3 zW?C;XQCHZl7H`K5^BF!Q5X2^Mj93&0l_O3Ea3!Ave|ixx+~bS@Iv18v2ctpSt4zO{ zp#7pj!AtDmti$T`e9{s^jf(ku&E|83JIJO5Qo9weT6g?@vX!{7)cNwymo1+u(YQ94 zopuz-L@|5=h8A!(g-MXgLJC0MA|CgQF8qlonnu#j z;uCeq9ny9QSD|p)9sp3ebgY3rk#y0DA(SHdh$DUm^?GI<>%e1?&}w(b zdip1;P2Z=1wM+$q=TgLP$}svd!vk+BZ@h<^4R=GS2+sri7Z*2f`9 z5_?i)xj?m#pSVchk-SR!2&uNhzEi+#5t1Z$o0PoLGz*pT64%+|Wa+rd5Z}60(j?X= z{NLjtgRb|W?CUADqOS@(*MA-l|E342NxRaxLTDqsOyfWWe%N(jjBh}G zm7WPel6jXijaTiNita+z(5GCO0NM=Melxud57PP^d_U## zbA;9iVi<@wr0DGB8=T9Ab#2K_#zi=$igyK48@;V|W`fg~7;+!q8)aCOo{HA@vpSy-4`^!ze6-~8|QE||hC{ICKllG9fbg_Y7v z$jn{00!ob3!@~-Z%!rSZ0JO#@>|3k10mLK0JRKP-Cc8UYFu>z93=Ab-r^oL2 zl`-&VBh#=-?{l1TatC;VweM^=M7-DUE>m+xO7Xi6vTEsReyLs8KJ+2GZ&rxw$d4IT zPXy6pu^4#e;;ZTsgmG+ZPx>piodegkx2n0}SM77+Y*j^~ICvp#2wj^BuqRY*&cjmL zcKp78aZt>e{3YBb4!J_2|K~A`lN=u&5j!byw`1itV(+Q_?RvV7&Z5XS1HF)L2v6ji z&kOEPmv+k_lSXb{$)of~(BkO^py&7oOzpjdG>vI1kcm_oPFHy38%D4&A4h_CSo#lX z2#oqMCTEP7UvUR3mwkPxbl8AMW(e{ARi@HCYLPSHE^L<1I}OgZD{I#YH#GKnpRmW3 z2jkz~Sa(D)f?V?$gNi?6)Y;Sm{&?~2p=0&BUl_(@hYeX8YjaRO=IqO7neK0RsSNdYjD zaw$g2sG(>JR=8Iz1SK4`*kqd_3-?;_BIcaaMd^}<@MYbYisWZm2C2|Np_l|8r9yM|JkUngSo@?wci(7&O9a z%|V(4C1c9pps0xxzPbXH=}QTxc2rr7fXk$9`a6TbWKPCz&p=VsB8^W96W=BsB|7bc zf(QR8&Ktj*iz)wK&mW`#V%4XTM&jWNnDF56O+2bo<3|NyUhQ%#OZE8$Uv2a@J>D%t zMVMiHh?es!Ex19q&6eC&L=XDU_BA&uR^^w>fpz2_`U87q_?N2y;!Z!bjoeKrzfC)} z?m^PM=(z{%n9K`p|7Bz$LuC7!>tFOuN74MFELm}OD9?%jpT>38J;=1Y-VWtZAscaI z_8jUZ#GwWz{JqvGEUmL?G#l5E=*m>`cY?m*XOc*yOCNtpuIGD+Z|kn4Xww=BLrNYS zGO=wQh}Gtr|7DGXLF%|`G>J~l{k^*{;S-Zhq|&HO7rC_r;o`gTB7)uMZ|WWIn@e0( zX$MccUMv3ABg^$%_lNrgU{EVi8O^UyGHPNRt%R!1#MQJn41aD|_93NsBQhP80yP<9 zG4(&0u7AtJJXLPcqzjv`S~5;Q|5TVGccN=Uzm}K{v)?f7W!230C<``9(64}D2raRU zAW5bp%}VEo{4Rko`bD%Ehf=0voW?-4Mk#d3_pXTF!-TyIt6U+({6OXWVAa;s-`Ta5 zTqx&8msH3+DLrVmQOTBOAj=uoxKYT3DS1^zBXM?1W+7gI!aQNPYfUl{3;PzS9*F7g zWJN8x?KjBDx^V&6iCY8o_gslO16=kh(|Gp)kz8qlQ`dzxQv;)V&t+B}wwdi~uBs4? zu~G|}y!`3;8#vIMUdyC7YEx6bb^1o}G!Jky4cN?BV9ejBfN<&!4M)L&lRKiuMS#3} z_B}Nkv+zzxhy{dYCW$oGC&J(Ty&7%=5B$sD0bkuPmj7g>|962`(Q{ZZMDv%YMuT^KweiRDvYTEop3IgFv#)(w>1 zSzH>J`q!LK)c(AK>&Ib)A{g`Fdykxqd`Yq@yB}E{gnQV$K!}RsgMGWqC3DKE(=!{}ekB3+(1?g}xF>^icEJbc z5bdxAPkW90atZT+&*7qoLqL#p=>t-(-lsnl2XMpZcYeW|o|a322&)yO_8p(&Sw{|b zn(tY$xn5yS$DD)UYS%sP?c|z>1dp!QUD)l;aW#`%qMtQJjE!s2z`+bTSZmLK7SvCR z=@I4|U^sCwZLQSfd*ACw9B@`1c1|&i^W_OD(570SDLK`MD0wTiR8|$7+%{cF&){$G zU~|$^Ed?TIxyw{1$e|D$050n8AjJvvOWhLtLHbSB|HIfjMp+gu>DraHZJRrdO53(= z+o-f{+qNog+qSLB%KY;5>Av6X(>-qYk3IIEwZ5~6a+P9lMpC^ z8CJ0q>rEpjlsxCvJm=kms@tlN4+sv}He`xkr`S}bGih4t`+#VEIt{1veE z{ZLtb_pSbcfcYPf4=T1+|BtR!x5|X#x2TZEEkUB6kslKAE;x)*0x~ES0kl4Dex4e- zT2P~|lT^vUnMp{7e4OExfxak0EE$Hcw;D$ehTV4a6hqxru0$|Mo``>*a5=1Ym0u>BDJKO|=TEWJ5jZu!W}t$Kv{1!q`4Sn7 zrxRQOt>^6}Iz@%gA3&=5r;Lp=N@WKW;>O!eGIj#J;&>+3va^~GXRHCY2}*g#9ULab zitCJt-OV0*D_Q3Q`p1_+GbPxRtV_T`jyATjax<;zZ?;S+VD}a(aN7j?4<~>BkHK7bO8_Vqfdq1#W&p~2H z&w-gJB4?;Q&pG9%8P(oOGZ#`!m>qAeE)SeL*t8KL|1oe;#+uOK6w&PqSDhw^9-&Fa zuEzbi!!7|YhlWhqmiUm!muO(F8-F7|r#5lU8d0+=;<`{$mS=AnAo4Zb^{%p}*gZL! zeE!#-zg0FWsSnablw!9$<&K(#z!XOW z;*BVx2_+H#`1b@>RtY@=KqD)63brP+`Cm$L1@ArAddNS1oP8UE$p05R=bvZoYz+^6 z<)!v7pRvi!u_-V?!d}XWQR1~0q(H3{d^4JGa=W#^Z<@TvI6J*lk!A zZ*UIKj*hyO#5akL*Bx6iPKvR3_2-^2mw|Rh-3O_SGN3V9GRo52Q;JnW{iTGqb9W99 z7_+F(Op6>~3P-?Q8LTZ-lwB}xh*@J2Ni5HhUI3`ct|*W#pqb>8i*TXOLn~GlYECIj zhLaa_rBH|1jgi(S%~31Xm{NB!30*mcsF_wgOY2N0XjG_`kFB+uQuJbBm3bIM$qhUyE&$_u$gb zpK_r{99svp3N3p4yHHS=#csK@j9ql*>j0X=+cD2dj<^Wiu@i>c_v zK|ovi7}@4sVB#bzq$n3`EgI?~xDmkCW=2&^tD5RuaSNHf@Y!5C(Is$hd6cuyoK|;d zO}w2AqJPS`Zq+(mc*^%6qe>1d&(n&~()6-ZATASNPsJ|XnxelLkz8r1x@c2XS)R*H(_B=IN>JeQUR;T=i3<^~;$<+8W*eRKWGt7c#>N`@;#!`kZ!P!&{9J1>_g8Zj zXEXxmA=^{8A|3=Au+LfxIWra)4p<}1LYd_$1KI0r3o~s1N(x#QYgvL4#2{z8`=mXy zQD#iJ0itk1d@Iy*DtXw)Wz!H@G2St?QZFz zVPkM%H8Cd2EZS?teQN*Ecnu|PrC!a7F_XX}AzfZl3fXfhBtc2-)zaC2eKx*{XdM~QUo4IwcGgVdW69 z1UrSAqqMALf^2|(I}hgo38l|Ur=-SC*^Bo5ej`hb;C$@3%NFxx5{cxXUMnTyaX{>~ zjL~xm;*`d08bG_K3-E+TI>#oqIN2=An(C6aJ*MrKlxj?-;G zICL$hi>`F%{xd%V{$NhisHSL~R>f!F7AWR&7b~TgLu6!3s#~8|VKIX)KtqTH5aZ8j zY?wY)XH~1_a3&>#j7N}0az+HZ;is;Zw(Am{MX}YhDTe(t{ZZ;TG}2qWYO+hdX}vp9 z@uIRR8g#y~-^E`Qyem(31{H0&V?GLdq9LEOb2(ea#e-$_`5Q{T%E?W(6 z(XbX*Ck%TQM;9V2LL}*Tf`yzai{0@pYMwBu%(I@wTY!;kMrzcfq0w?X`+y@0ah510 zQX5SU(I!*Fag4U6a7Lw%LL;L*PQ}2v2WwYF(lHx_Uz2ceI$mnZ7*eZ?RFO8UvKI0H z9Pq-mB`mEqn6n_W9(s~Jt_D~j!Ln9HA)P;owD-l~9FYszs)oEKShF9Zzcmnb8kZ7% zQ`>}ki1kwUO3j~ zEmh140sOkA9v>j@#56ymn_RnSF`p@9cO1XkQy6_Kog?0ivZDb`QWOX@tjMd@^Qr(p z!sFN=A)QZm!sTh(#q%O{Ovl{IxkF!&+A)w2@50=?a-+VuZt6On1;d4YtUDW{YNDN_ zG@_jZi1IlW8cck{uHg^g=H58lPQ^HwnybWy@@8iw%G! zwB9qVGt_?~M*nFAKd|{cGg+8`+w{j_^;nD>IrPf-S%YjBslSEDxgKH{5p)3LNr!lD z4ii)^%d&cCXIU7UK?^ZQwmD(RCd=?OxmY(Ko#+#CsTLT;p#A%{;t5YpHFWgl+@)N1 zZ5VDyB;+TN+g@u~{UrWrv)&#u~k$S&GeW)G{M#&Di)LdYk?{($Cq zZGMKeYW)aMtjmKgvF0Tg>Mmkf9IB#2tYmH-s%D_9y3{tfFmX1BSMtbe<(yqAyWX60 zzkgSgKb3c{QPG2MalYp`7mIrYg|Y<4Jk?XvJK)?|Ecr+)oNf}XLPuTZK%W>;<|r+% zTNViRI|{sf1v7CsWHvFrkQ$F7+FbqPQ#Bj7XX=#M(a~9^80}~l-DueX#;b}Ajn3VE z{BWI}$q{XcQ3g{(p>IOzFcAMDG0xL)H%wA)<(gl3I-oVhK~u_m=hAr&oeo|4lZbf} z+pe)c34Am<=z@5!2;_lwya;l?xV5&kWe}*5uBvckm(d|7R>&(iJNa6Y05SvlZcWBlE{{%2- z`86)Y5?H!**?{QbzGG~|k2O%eA8q=gxx-3}&Csf6<9BsiXC)T;x4YmbBIkNf;0Nd5 z%whM^!K+9zH>on_<&>Ws?^v-EyNE)}4g$Fk?Z#748e+GFp)QrQQETx@u6(1fk2!(W zWiCF~MomG*y4@Zk;h#2H8S@&@xwBIs|82R*^K(i*0MTE%Rz4rgO&$R zo9Neb;}_ulaCcdn3i17MO3NxzyJ=l;LU*N9ztBJ30j=+?6>N4{9YXg$m=^9@Cl9VY zbo^{yS@gU=)EpQ#;UIQBpf&zfCA;00H-ee=1+TRw@(h%W=)7WYSb5a%$UqNS@oI@= zDrq|+Y9e&SmZrH^iA>Of8(9~Cf-G(P^5Xb%dDgMMIl8gk6zdyh`D3OGNVV4P9X|EvIhplXDld8d z^YWtYUz@tpg*38Xys2?zj$F8%ivA47cGSl;hjD23#*62w3+fwxNE7M7zVK?x_`dBSgPK zWY_~wF~OEZi9|~CSH8}Xi>#8G73!QLCAh58W+KMJJC81{60?&~BM_0t-u|VsPBxn* zW7viEKwBBTsn_A{g@1!wnJ8@&h&d>!qAe+j_$$Vk;OJq`hrjzEE8Wjtm)Z>h=*M25 zOgETOM9-8xuuZ&^@rLObtcz>%iWe%!uGV09nUZ*nxJAY%&KAYGY}U1WChFik7HIw% zZP$3Bx|TG_`~19XV7kfi2GaBEhKap&)Q<9`aPs#^!kMjtPb|+-fX66z3^E)iwyXK7 z8)_p<)O{|i&!qxtgBvWXx8*69WO$5zACl++1qa;)0zlXf`eKWl!0zV&I`8?sG)OD2Vy?reNN<{eK+_ za4M;Hh%&IszR%)&gpgRCP}yheQ+l#AS-GnY81M!kzhWxIR?PW`G3G?} z$d%J28uQIuK@QxzGMKU_;r8P0+oIjM+k)&lZ39i#(ntY)*B$fdJnQ3Hw3Lsi8z&V+ zZly2}(Uzpt2aOubRjttzqrvinBFH4jrN)f0hy)tj4__UTwN)#1fj3-&dC_Vh7}ri* zfJ=oqLMJ-_<#rwVyN}_a-rFBe2>U;;1(7UKH!$L??zTbbzP#bvyg7OQBGQklJ~DgP zd<1?RJ<}8lWwSL)`jM53iG+}y2`_yUvC!JkMpbZyb&50V3sR~u+lok zT0uFRS-yx@8q4fPRZ%KIpLp8R#;2%c&Ra4p(GWRT4)qLaPNxa&?8!LRVdOUZ)2vrh zBSx&kB%#Y4!+>~)<&c>D$O}!$o{<1AB$M7-^`h!eW;c(3J~ztoOgy6Ek8Pwu5Y`Xion zFl9fb!k2`3uHPAbd(D^IZmwR5d8D$495nN2`Ue&`W;M-nlb8T-OVKt|fHk zBpjX$a(IR6*-swdNk@#}G?k6F-~c{AE0EWoZ?H|ZpkBxqU<0NUtvubJtwJ1mHV%9v?GdDw; zAyXZiD}f0Zdt-cl9(P1la+vQ$Er0~v}gYJVwQazv zH#+Z%2CIfOf90fNMGos|{zf&N`c0@x0N`tkFv|_9af3~<0z@mnf*e;%r*Fbuwl-IW z{}B3=(mJ#iwLIPiUP`J3SoP~#)6v;aRXJ)A-pD2?_2_CZ#}SAZ<#v7&Vk6{*i(~|5 z9v^nC`T6o`CN*n%&9+bopj^r|E(|pul;|q6m7Tx+U|UMjWK8o-lBSgc3ZF=rP{|l9 zc&R$4+-UG6i}c==!;I#8aDIbAvgLuB66CQLRoTMu~jdw`fPlKy@AKYWS-xyZzPg&JRAa@m-H43*+ne!8B7)HkQY4 zIh}NL4Q79a-`x;I_^>s$Z4J4-Ngq=XNWQ>yAUCoe&SMAYowP>r_O}S=V+3=3&(O=h zNJDYNs*R3Y{WLmBHc?mFEeA4`0Y`_CN%?8qbDvG2m}kMAiqCv`_BK z_6a@n`$#w6Csr@e2YsMx8udNWtNt=kcqDZdWZ-lGA$?1PA*f4?X*)hjn{sSo8!bHz zb&lGdAgBx@iTNPK#T_wy`KvOIZvTWqSHb=gWUCKXAiB5ckQI`1KkPx{{%1R*F2)Oc z(9p@yG{fRSWE*M9cdbrO^)8vQ2U`H6M>V$gK*rz!&f%@3t*d-r3mSW>D;wYxOhUul zk~~&ip5B$mZ~-F1orsq<|1bc3Zpw6)Ws5;4)HilsN;1tx;N6)tuePw& z==OlmaN*ybM&-V`yt|;vDz(_+UZ0m&&9#{9O|?0I|4j1YCMW;fXm}YT$0%EZ5^YEI z4i9WV*JBmEU{qz5O{#bs`R1wU%W$qKx?bC|e-iS&d*Qm7S=l~bMT{~m3iZl+PIXq{ zn-c~|l)*|NWLM%ysfTV-oR0AJ3O>=uB-vpld{V|cWFhI~sx>ciV9sPkC*3i0Gg_9G!=4ar*-W?D9)?EFL1=;O+W8}WGdp8TT!Fgv z{HKD`W>t(`Cds_qliEzuE!r{ihwEv1l5o~iqlgjAyGBi)$%zNvl~fSlg@M=C{TE;V zQkH`zS8b&!ut(m)%4n2E6MB>p*4(oV>+PT51#I{OXs9j1vo>9I<4CL1kv1aurV*AFZ^w_qfVL*G2rG@D2 zrs87oV3#mf8^E5hd_b$IXfH6vHe&lm@7On~Nkcq~YtE!}ad~?5*?X*>y`o;6Q9lkk zmf%TYonZM`{vJg$`lt@MXsg%*&zZZ0uUSse8o=!=bfr&DV)9Y6$c!2$NHyYAQf*Rs zk{^?gl9E z5Im8wlAsvQ6C2?DyG@95gUXZ3?pPijug25g;#(esF_~3uCj3~94}b*L>N2GSk%Qst z=w|Z>UX$m!ZOd(xV*2xvWjN&c5BVEdVZ0wvmk)I+YxnyK%l~caR=7uNQ=+cnNTLZ@&M!I$Mj-r{!P=; z`C2)D=VmvK8@T5S9JZoRtN!S*D_oqOxyy!q6Zk|~4aT|*iRN)fL)c>-yycR>-is0X zKrko-iZw(f(!}dEa?hef5yl%p0-v-8#8CX8!W#n2KNyT--^3hq6r&`)5Y@>}e^4h- zlPiDT^zt}Ynk&x@F8R&=)k8j$=N{w9qUcIc&)Qo9u4Y(Ae@9tA`3oglxjj6c{^pN( zQH+Uds2=9WKjH#KBIwrQI%bbs`mP=7V>rs$KG4|}>dxl_k!}3ZSKeEen4Iswt96GGw`E6^5Ov)VyyY}@itlj&sao|>Sb5 zeY+#1EK(}iaYI~EaHQkh7Uh>DnzcfIKv8ygx1Dv`8N8a6m+AcTa-f;17RiEed>?RT zk=dAksmFYPMV1vIS(Qc6tUO+`1jRZ}tcDP? zt)=7B?yK2RcAd1+Y!$K5*ds=SD;EEqCMG6+OqPoj{&8Y5IqP(&@zq@=A7+X|JBRi4 zMv!czlMPz)gt-St2VZwDD=w_S>gRpc-g zUd*J3>bXeZ?Psjohe;z7k|d<*T21PA1i)AOi8iMRwTBSCd0ses{)Q`9o&p9rsKeLaiY zluBw{1r_IFKR76YCAfl&_S1*(yFW8HM^T()&p#6y%{(j7Qu56^ZJx1LnN`-RTwimdnuo*M8N1ISl+$C-%=HLG-s} zc99>IXRG#FEWqSV9@GFW$V8!{>=lSO%v@X*pz*7()xb>=yz{E$3VE;e)_Ok@A*~El zV$sYm=}uNlUxV~6e<6LtYli1!^X!Ii$L~j4e{sI$tq_A(OkGquC$+>Rw3NFObV2Z)3Rt~Jr{oYGnZaFZ^g5TDZlg;gaeIP} z!7;T{(9h7mv{s@piF{-35L=Ea%kOp;^j|b5ZC#xvD^^n#vPH=)lopYz1n?Kt;vZmJ z!FP>Gs7=W{sva+aO9S}jh0vBs+|(B6Jf7t4F^jO3su;M13I{2rd8PJjQe1JyBUJ5v zcT%>D?8^Kp-70bP8*rulxlm)SySQhG$Pz*bo@mb5bvpLAEp${?r^2!Wl*6d7+0Hs_ zGPaC~w0E!bf1qFLDM@}zso7i~(``)H)zRgcExT_2#!YOPtBVN5Hf5~Ll3f~rWZ(UsJtM?O*cA1_W0)&qz%{bDoA}{$S&-r;0iIkIjbY~ zaAqH45I&ALpP=9Vof4OapFB`+_PLDd-0hMqCQq08>6G+C;9R~}Ug_nm?hhdkK$xpI zgXl24{4jq(!gPr2bGtq+hyd3%Fg%nofK`psHMs}EFh@}sdWCd!5NMs)eZg`ZlS#O0 zru6b8#NClS(25tXqnl{|Ax@RvzEG!+esNW-VRxba(f`}hGoqci$U(g30i}2w9`&z= zb8XjQLGN!REzGx)mg~RSBaU{KCPvQx8)|TNf|Oi8KWgv{7^tu}pZq|BS&S<53fC2K4Fw6>M^s$R$}LD*sUxdy6Pf5YKDbVet;P!bw5Al-8I1Nr(`SAubX5^D9hk6$agWpF}T#Bdf{b9-F#2WVO*5N zp+5uGgADy7m!hAcFz{-sS0kM7O)qq*rC!>W@St~^OW@R1wr{ajyYZq5H!T?P0e+)a zaQ%IL@X_`hzp~vRH0yUblo`#g`LMC%9}P;TGt+I7qNcBSe&tLGL4zqZqB!Bfl%SUa z6-J_XLrnm*WA`34&mF+&e1sPCP9=deazrM=Pc4Bn(nV;X%HG^4%Afv4CI~&l!Sjzb z{rHZ3od0!Al{}oBO>F*mOFAJrz>gX-vs!7>+_G%BB(ljWh$252j1h;9p~xVA=9_`P z5KoFiz96_QsTK%B&>MSXEYh`|U5PjX1(+4b#1PufXRJ*uZ*KWdth1<0 zsAmgjT%bowLyNDv7bTUGy|g~N34I-?lqxOUtFpTLSV6?o?<7-UFy*`-BEUsrdANh} zBWkDt2SAcGHRiqz)x!iVoB~&t?$yn6b#T=SP6Ou8lW=B>=>@ik93LaBL56ub`>Uo!>0@O8?e)$t(sgy$I z6tk3nS@yFFBC#aFf?!d_3;%>wHR;A3f2SP?Na8~$r5C1N(>-ME@HOpv4B|Ty7%jAv zR}GJwsiJZ5@H+D$^Cwj#0XA_(m^COZl8y7Vv(k=iav1=%QgBOVzeAiw zaDzzdrxzj%sE^c9_uM5D;$A_7)Ln}BvBx^=)fO+${ou%B*u$(IzVr-gH3=zL6La;G zu0Kzy5CLyNGoKRtK=G0-w|tnwI)puPDOakRzG(}R9fl7#<|oQEX;E#yCWVg95 z;NzWbyF&wGg_k+_4x4=z1GUcn6JrdX4nOVGaAQ8#^Ga>aFvajQN{!+9rgO-dHP zIp@%&ebVg}IqnRWwZRTNxLds+gz2@~VU(HI=?Epw>?yiEdZ>MjajqlO>2KDxA>)cj z2|k%dhh%d8SijIo1~20*5YT1eZTDkN2rc^zWr!2`5}f<2f%M_$to*3?Ok>e9$X>AV z2jYmfAd)s|(h?|B(XYrIfl=Wa_lBvk9R1KaP{90-z{xKi+&8=dI$W0+qzX|ZovWGOotP+vvYR(o=jo?k1=oG?%;pSqxcU* zWVGVMw?z__XQ9mnP!hziHC`ChGD{k#SqEn*ph6l46PZVkm>JF^Q{p&0=MKy_6apts z`}%_y+Tl_dSP(;Ja&sih$>qBH;bG;4;75)jUoVqw^}ee=ciV;0#t09AOhB^Py7`NC z-m+ybq1>_OO+V*Z>dhk}QFKA8V?9Mc4WSpzj{6IWfFpF7l^au#r7&^BK2Ac7vCkCn{m0uuN93Ee&rXfl1NBY4NnO9lFUp zY++C1I;_{#OH#TeP2Dp?l4KOF8ub?m6zE@XOB5Aiu$E~QNBM@;r+A5mF2W1-c7>ex zHiB=WJ&|`6wDq*+xv8UNLVUy4uW1OT>ey~Xgj@MMpS@wQbHAh>ysYvdl-1YH@&+Q! z075(Qd4C!V`9Q9jI4 zSt{HJRvZec>vaL_brKhQQwbpQd4_Lmmr0@1GdUeU-QcC{{8o=@nwwf>+dIKFVzPriGNX4VjHCa zTbL9w{Y2V87c2ofX%`(48A+4~mYTiFFl!e{3K^C_k%{&QTsgOd0*95KmWN)P}m zTRr{`f7@=v#+z_&fKYkQT!mJn{*crj%ZJz#(+c?>cD&2Lo~FFAWy&UG*Op^pV`BR^I|g?T>4l5;b|5OQ@t*?_Slp`*~Y3`&RfKD^1uLezIW(cE-Dq2z%I zBi8bWsz0857`6e!ahet}1>`9cYyIa{pe53Kl?8|Qg2RGrx@AlvG3HAL-^9c^1GW;)vQt8IK+ zM>!IW*~682A~MDlyCukldMd;8P|JCZ&oNL(;HZgJ>ie1PlaInK7C@Jg{3kMKYui?e!b`(&?t6PTb5UPrW-6DVU%^@^E`*y-Fd(p|`+JH&MzfEq;kikdse ziFOiDWH(D< zyV7Rxt^D0_N{v?O53N$a2gu%1pxbeK;&ua`ZkgSic~$+zvt~|1Yb=UfKJW2F7wC^evlPf(*El+#}ZBy0d4kbVJsK- z05>;>?HZO(YBF&v5tNv_WcI@O@LKFl*VO?L(!BAd!KbkVzo;v@~3v`-816GG?P zY+H3ujC>5=Am3RIZDdT#0G5A6xe`vGCNq88ZC1aVXafJkUlcYmHE^+Z{*S->ol%-O znm9R0TYTr2w*N8Vs#s-5=^w*{Y}qp5GG)Yt1oLNsH7y~N@>Eghms|K*Sdt_u!&I}$ z+GSdFTpbz%KH+?B%Ncy;C`uW6oWI46(tk>r|5|-K6)?O0d_neghUUOa9BXHP*>vi; z={&jIGMn-92HvInCMJcyXwHTJ42FZp&Wxu+9Rx;1x(EcIQwPUQ@YEQQ`bbMy4q3hP zNFoq~Qd0=|xS-R}k1Im3;8s{BnS!iaHIMLx)aITl)+)?Yt#fov|Eh>}dv@o6R{tG>uHsy&jGmWN5+*wAik|78(b?jtysPHC#e+Bzz~V zS3eEXv7!Qn4uWi!FS3B?afdD*{fr9>B~&tc671fi--V}~E4un;Q|PzZRwk-azprM$4AesvUb5`S`(5x#5VJ~4%ET6&%GR$}muHV-5lTsCi_R|6KM(g2PCD@|yOpKluT zakH!1V7nKN)?6JmC-zJoA#ciFux8!)ajiY%K#RtEg$gm1#oKUKX_Ms^%hvKWi|B=~ zLbl-L)-=`bfhl`>m!^sRR{}cP`Oim-{7}oz4p@>Y(FF5FUEOfMwO!ft6YytF`iZRq zfFr{!&0Efqa{1k|bZ4KLox;&V@ZW$997;+Ld8Yle91he{BfjRhjFTFv&^YuBr^&Pe zswA|Bn$vtifycN8Lxr`D7!Kygd7CuQyWqf}Q_PM}cX~S1$-6xUD%-jrSi24sBTFNz(Fy{QL2AmNbaVggWOhP;UY4D>S zqKr!UggZ9Pl9Nh_H;qI`-WoH{ceXj?m8y==MGY`AOJ7l0Uu z)>M%?dtaz2rjn1SW3k+p`1vs&lwb%msw8R!5nLS;upDSxViY98IIbxnh{}mRfEp=9 zbrPl>HEJeN7J=KnB6?dwEA6YMs~chHNG?pJsEj#&iUubdf3JJwu=C(t?JpE6xMyhA3e}SRhunDC zn-~83*9=mADUsk^sCc%&&G1q5T^HR9$P#2DejaG`Ui*z1hI#h7dwpIXg)C{8s< z%^#@uQRAg-$z&fmnYc$Duw63_Zopx|n{Bv*9Xau{a)2%?H<6D>kYY7_)e>OFT<6TT z0A}MQLgXbC2uf`;67`mhlcUhtXd)Kbc$PMm=|V}h;*_%vCw4L6r>3Vi)lE5`8hkSg zNGmW-BAOO)(W((6*e_tW&I>Nt9B$xynx|sj^ux~?q?J@F$L4;rnm_xy8E*JYwO-02u9_@@W0_2@?B@1J{y~Q39N3NX^t7#`=34Wh)X~sU&uZWgS1Z09%_k|EjA4w_QqPdY`oIdv$dJZ;(!k)#U8L+|y~gCzn+6WmFt#d{OUuKHqh1-uX_p*Af8pFYkYvKPKBxyid4KHc}H` z*KcyY;=@wzXYR{`d{6RYPhapShXIV?0cg_?ahZ7do)Ot#mxgXYJYx}<%E1pX;zqHd zf!c(onm{~#!O$2`VIXezECAHVd|`vyP)Uyt^-075X@NZDBaQt<>trA3nY-Dayki4S zZ^j6CCmx1r46`4G9794j-WC0&R9(G7kskS>=y${j-2;(BuIZTLDmAyWTG~`0)Bxqk zd{NkDe9ug|ms@0A>JVmB-IDuse9h?z9nw!U6tr7t-Lri5H`?TjpV~8(gZWFq4Vru4 z!86bDB;3lpV%{rZ`3gtmcRH1hjj!loI9jN>6stN6A*ujt!~s!2Q+U1(EFQEQb(h4E z6VKuRouEH`G6+8Qv2C)K@^;ldIuMVXdDDu}-!7FS8~k^&+}e9EXgx~)4V4~o6P^52 z)a|`J-fOirL^oK}tqD@pqBZi_;7N43%{IQ{v&G9^Y^1?SesL`;Z(dt!nn9Oj5Odde%opv&t zxJ><~b#m+^KV&b?R#)fRi;eyqAJ_0(nL*61yPkJGt;gZxSHY#t>ATnEl-E%q$E16% zZdQfvhm5B((y4E3Hk6cBdwGdDy?i5CqBlCVHZr-rI$B#>Tbi4}Gcvyg_~2=6O9D-8 zY2|tKrNzbVR$h57R?Pe+gUU_il}ZaWu|Az#QO@};=|(L-RVf0AIW zq#pO+RfM7tdV`9lI6g;{qABNId`fG%U9Va^ravVT^)CklDcx)YJKeJdGpM{W1v8jg z@&N+mR?BPB=K1}kNwXk_pj44sd>&^;d!Z~P>O78emE@Qp@&8PyB^^4^2f7e)gekMv z2aZNvP@;%i{+_~>jK7*2wQc6nseT^n6St9KG#1~Y@$~zR_=AcO2hF5lCoH|M&c{vR zSp(GRVVl=T*m~dIA;HvYm8HOdCkW&&4M~UDd^H)`p__!4k+6b)yG0Zcek8OLw$C^K z3-BbLiG_%qX|ZYpXJ$(c@aa7b4-*IQkDF}=gZSV`*ljP|5mWuHSCcf$5qqhZTv&P?I$z^>}qP(q!Aku2yA5vu38d8x*q{6-1`%PrE_r0-9Qo?a#7Zbz#iGI7K<(@k^|i4QJ1H z4jx?{rZbgV!me2VT72@nBjucoT zUM9;Y%TCoDop?Q5fEQ35bCYk7!;gH*;t9t-QHLXGmUF;|vm365#X)6b2Njsyf1h9JW#x$;@x5Nx2$K$Z-O3txa%;OEbOn6xBzd4n4v)Va=sj5 z%rb#j7{_??Tjb8(Hac<^&s^V{yO-BL*uSUk2;X4xt%NC8SjO-3?;Lzld{gM5A=9AV z)DBu-Z8rRvXXwSVDH|dL-3FODWhfe1C_iF``F05e{dl(MmS|W%k-j)!7(ARkV?6r~ zF=o42y+VapxdZn;GnzZfGu<6oG-gQ7j7Zvgo7Am@jYxC2FpS@I;Jb%EyaJDBQC(q% zKlZ}TVu!>;i3t~OAgl@QYy1X|T~D{HOyaS*Bh}A}S#a9MYS{XV{R-|niEB*W%GPW! zP^NU(L<}>Uab<;)#H)rYbnqt|dOK(-DCnY==%d~y(1*{D{Eo1cqIV8*iMfx&J*%yh zx=+WHjt0q2m*pLx8=--UqfM6ZWjkev>W-*}_*$Y(bikH`#-Gn#!6_ zIA&kxn;XYI;eN9yvqztK-a113A%97in5CL5Z&#VsQ4=fyf&3MeKu70)(x^z_uw*RG zo2Pv&+81u*DjMO6>Mrr7vKE2CONqR6C0(*;@4FBM;jPIiuTuhQ-0&C)JIzo_k>TaS zN_hB;_G=JJJvGGpB?uGgSeKaix~AkNtYky4P7GDTW6{rW{}V9K)Cn^vBYKe*OmP!; zohJs=l-0sv5&pL6-bowk~(swtdRBZQHh8)m^r2+qTtZ zt4m$B?OQYNyfBA0E)g28a*{)a=%%f-?{F;++-Xs#5|7kSHTD*E9@$V ztE%7zX4A(L`n)FY8Y4pOnKC|Pf)j$iR#yP;V0+|Hki+D;t4I4BjkfdYliK9Gf6RYw z;3px$Ud5aTd`yq$N7*WOs!{X91hZZ;AJ9iQOH%p;v$R%OQum_h#rq9*{ve(++|24z zh2P;{-Z?u#rOqd0)D^_Ponv(Y9KMB9#?}nJdUX&r_rxF0%3__#8~ZwsyrSPmtWY27 z-54ZquV2t_W!*+%uwC=h-&_q~&nQer0(FL74to%&t^byl^C?wTaZ-IS9OssaQFP)1 zAov0o{?IRAcCf+PjMWSdmP42gysh|c9Ma&Q^?_+>>+-yrC8WR;*XmJ;>r9v*>=W}tgWG;WIt{~L8`gk8DP{dSdG z4SDM7g5ahMHYHHk*|mh9{AKh-qW7X+GEQybJt9A@RV{gaHUAva+=lSroK^NUJYEiL z?X6l9ABpd)9zzA^;FdZ$QQs#uD@hdcaN^;Q=AXlbHv511Meye`p>P4Y2nblEDEeZo}-$@g&L98Aih6tgLz--${eKTxymIipy0xSYgZZ zq^yyS4yNPTtPj-sM?R8@9Q1gtXPqv{$lb5i|C1yymwnGdfYV3nA-;5!Wl zD0fayn!B^grdE?q^}ba{-LIv*Z}+hZm_F9c$$cW!bx2DgJD&6|bBIcL@=}kQA1^Eh zXTEznqk)!!IcTl>ey?V;X8k<+C^DRA{F?T*j0wV`fflrLBQq!l7cbkAUE*6}WabyF zgpb+|tv=aWg0i}9kBL8ZCObYqHEycr5tpc-$|vdvaBsu#lXD@u_e1iL z{h>xMRS0a7KvW?VttrJFpX^5DC4Bv4cp6gNG6#8)7r7IxXfSNSp6)_6tZ4l>(D+0I zPhU)N!sKywaBusHdVE!yo5$20JAU8V_XcW{QmO!p*~ns8{2~bhjydnmA&=r zX9NSM9QYogYMDZ~kS#Qx`mt>AmeR3p@K$`fbJ%LQ1c5lEOz<%BS<}2DL+$>MFcE%e zlxC)heZ7#i80u?32eOJI9oQRz0z;JW@7Th4q}YmQ-`Z?@y3ia^_)7f37QMwDw~<-@ zT)B6fftmK_6YS!?{uaj5lLxyR++u*ZY2Mphm5cd7PA5=%rd)95hJ9+aGSNfjy>Ylc zoI0nGIT3sKmwX8h=6CbvhVO+ehFIR155h8iRuXZx^cW>rq5K4z_dvM#hRER=WR@THs%WELI9uYK9HN44Em2$#@k)hD zicqRPKV#yB;UlcsTL_}zCMK0T;eXHfu`y2(dfwm(v)IBbh|#R>`2cot{m7}8_X&oD zr@94PkMCl%d3FsC4pil=#{3uv^+)pvxfwmPUr)T)T|GcZVD$wVj$mjkjDs`5cm8N! zXVq2CvL;gWGpPI4;9j;2&hS*o+LNp&C5Ac=OXx*W5y6Z^az)^?G0)!_iAfjH5wiSE zD(F}hQZB#tF5iEx@0sS+dP70DbZ*<=5X^)Pxo^8aKzOzuyc2rq=<0-k;Y_ID1>9^v z+)nc36}?>jen*1%OX3R*KRASj${u$gZ$27Hpcj=95kK^aLzxhW6jj_$w6}%#1*$5D zG1H_vYFrCSwrRqYw*9<}OYAOQT)u%9lC`$IjZV<4`9Sc;j{Qv_6+uHrYifK&On4V_7yMil!0Yv55z@dFyD{U@Sy>|vTX=P_( zRm<2xj*Z}B30VAu@0e+}at*y?wXTz|rPalwo?4ZZc>hS0Ky6~mi@kv#?xP2a;yt?5=(-CqvP_3&$KdjB7Ku;# z`GLE*jW1QJB5d&E?IJO?1+!Q8HQMGvv^RuFoi=mM4+^tOqvX%X&viB%Ko2o-v4~~J z267ui;gsW?J=qS=D*@*xJvAy3IOop5bEvfR4MZC>9Y4Z$rGI|EHNNZ7KX;Ix{xSvm z-)Cau-xuTm|7`4kUdXvd_d^E=po(76ELfq5OgxIt3aqDy#zBfIy-5<3gpn{Ce`-ha z<;6y@{Bgqw?c~h*&j{FozQCh=`Lv-5Iw!KdSt;%GDOq%=(V!dJ-}|}|0o5G2kJj6{ z`jCSPs$9Fe8O(+qALZiJ$WtR=<@GvsdM)IJ`7XrBfW0iyYE#Vy^e@zbysg*B5Z_kSL6<)vqoaH zQ{!9!*{e9UZo^h+qZ`T@LfVwAEwc&+9{C8c%oj41q#hyn<&zA9IIur~V|{mmu`n5W z8)-Ou$YgjQ*PMIqHhZ_9E?(uoK0XM5aQkarcp}WT^7b^FC#^i>#8LGZ9puDuXUYas z7caX)V5U6uY-L5Wl%)j$qRkR;7@3T*N64YK_!`Fw=>CAwe~2loI1<>DZW&sb7Q)X;6E08&$h! z2=c1i4UOO{R4TmkTz+o9n`}+%d%blR6P;5{`qjtxlN$~I%tMMDCY`~e{+mRF!rj5( z3ywv)P_PUUqREu)TioPkg&5RKjY6z%pRxQPQ{#GNMTPag^S8(8l{!{WGNs2U1JA-O zq02VeYcArhTAS;v3);k(&6ayCH8SXN@r;1NQeJ*y^NHM+zOd;?t&c!Hq^SR_w6twGV8dl>j zjS+Zc&Yp7cYj&c1y3IxQ%*kWiYypvoh(k8g`HrY<_Bi-r%m-@SLfy-6mobxkWHxyS z>TtM2M4;Uqqy|+8Q++VcEq$PwomV1D4UzNA*Tgkg9#Gpz#~&iPf|Czx!J?qss?e|3 z4gTua75-P{2X7w9eeK3~GE0ip-D;%%gTi)8bR~Ez@)$gpuS~jZs`CrO5SR-Xy7bkA z89fr~mY}u4A$|r1$fe-;T{yJh#9Ime1iRu8eo?uY9@yqAU3P!rx~SsP;LTBL zeoMK(!;(Zt8313 z3)V)q_%eflKW?BnMZa}6E0c7t!$-mC$qt44OME5F(6B$E8w*TUN-h}0dOiXI+TH zYFrr&k1(yO(|J0vP|{22@Z}bxm@7BkjO)f)&^fv|?_JX+s)1*|7X7HH(W?b3QZ3!V|~m?8}uJsF>NvE4@fik zjyyh+U*tt`g6v>k9ub88a;ySvS1QawGn7}aaR**$rJA=a#eUT~ngUbJ%V=qsFIekLbv!YkqjTG{_$F;$w19$(ivIs*1>?2ka%uMOx@B9`LD zhm~)z@u4x*zcM1WhiX)!U{qOjJHt1xs{G1S?rYe)L)ntUu^-(o_dfqZu)}W(X%Uu| zN*qI@&R2fB#Jh|Mi+eMrZDtbNvYD3|v0Kx>E#Ss;Be*T$@DC!2A|mb%d}TTN3J+c= zu@1gTOXFYy972S+=C;#~)Z{Swr0VI5&}WYzH22un_Yg5o%f9fvV(`6!{C<(ZigQ2`wso)cj z9O12k)15^Wuv#rHpe*k5#4vb%c znP+Gjr<-p%01d<+^yrSoG?}F=eI8X;?=Fo2a~HUiJ>L!oE#9tXRp!adg-b9D;(6$E zeW0tH$US04zTX$OxM&X+2ip>KdFM?iG_fgOD-qB|uFng8*#Z5jgqGY=zLU?4!OlO#~YBTB9b9#~H@nqQ#5 z6bV));d?IJTVBC+79>rGuy1JgxPLy$dA7;_^^L)02m}XLjFR*qH`eI~+eJo(7D`LH z(W%lGnGK+Vk_3kyF*zpgO=1MxMg?hxe3}}YI>dVs8l}5eWjYu4=w6MWK09+05 zGdpa#$awd>Q|@aZa*z{5F3xy3n@E4YT9%TmMo0jxW59p0bI?&S}M+ z&^NG%rf7h*m9~p#b19|`wO5OMY-=^XT+=yrfGNpl<&~~FGsx_`IaFn+sEgF$hgOa~oAVAiu^a$jHcqkE=dj`ze z=axsfrzzh6VGD0x#6Ff=t%+VTiq!n6^gv*uIUD<9fOhvR;al5kcY${uunn}-!74<7 zmP^3cl-kyN(QY!!Z-^PY-OUkh=3ZWk6>le$_Q&xk4cgH{?i)C%2RM@pX5Q{jdSlo! zVau5v44cQX5|zQlQDt;dCg)oM0B<=P1CR!W%!^m$!{pKx;bn9DePJjWBX)q!`$;0K zqJIIyD#aK;#-3&Nf=&IhtbV|?ZGYHSphp~6th`p2rkw&((%kBV7<{siEOU7AxJj+FuRdDu$ zcmTW8usU_u!r)#jg|J=Gt{##7;uf4A5cdt6Y02}f(d2)z~ z)CH~gVAOwBLk$ZiIOn}NzDjvfw(w$u|BdCBI#)3xB-Ot?nz?iR38ayCm48M=_#9r7 zw8%pwQ<9mbEs5~_>pN3~#+Er~Q86J+2TDXM6umCbukd-X6pRIr5tF?VauT8jW> zY^#)log>jtJs2s3xoiPB7~8#1ZMv>Zx0}H58k-@H2huNyw~wsl0B8j)H5)H9c7y&i zp8^0;rKbxC1eEZ-#Qxvz)Xv$((8lK9I>BspPajluysw^f#t9P;OUis43mmEzX+lk* zc4T-Ms9_687GR+~QS#0~vxK#DSGN=a-m(@eZTqw2<+lN9>R~gK2)3;sT4%nI%Y|0m zX9SPR!>?~s=j5H4WMqeTW8QaLZ=1bWS5I3xZ&$(ypc=tHrv+hX@s)VG(tc!yvLM7n zshN=C#v={X1r;)xn0Pow_1eMhkn!{;x$BJ#PIz)m585&%cmzk;btQzZAN_^zis;n? z?6I~bN?s;7vg_dtoTc4A5Ow*Rb}No#UYl)sN|RmoYo}k^cKLXd8F`44?RrokkPvN5 ztUrx;U~B;jbE_qGd3n0j2i}A{enJvJ?gSF~NQj~EP5vM-w4@;QQ5n(Npic}XNW6B0 zq9F4T%6kp7qGhd0vpQcz+nMk8GOAmbz8Bt4@GtewGr6_>Xj>ge)SyfY}nu>Y!a@HoIx(StD zx`!>RT&}tpBL%nOF%7XIFW?n1AP*xthCMzhrU6G!U6?m4!CPWTvn#Yaoi_95CT2!L z|B=5zeRW30&ANGN>J9#GtCm&3SF6n4TqDz<-{@ZXkrkRDCpV$DwCtI^e&3i1A{Ar&JZtS^c+lyPa6 z%JJr42S_;eFC#M~bdtQePhOU32WDiZ4@H&af)z#$Y|hnQNb)8(3?1Ad>5uaZ1z zU~!jt3XUI@gpWb8tWTyH7DGvKvzYfqNIy3P{9vpwz_C-QL&`+8Io$F5PS-@YQJoEO z17D9P(+sXajWSH_8&C?fn>rTLX+(?KiwX#JNV)xE0!Q@>Tid$V2#r4y6fkph?YZ>^ z(o^q(0*P->3?I0cELXJn(N|#qTm6 zAPIL~n)m!50;*?5=MOOc4Wk;w(0c$(!e?vpV23S|n|Y7?nyc8)fD8t-KI&nTklH&BzqQ}D(1gH3P+5zGUzIjT~x`;e8JH=86&5&l-DP% z)F+Et(h|GJ?rMy-Zrf>Rv@<3^OrCJ1xv_N*_@-K5=)-jP(}h1Rts44H&ou8!G_C1E zhTfUDASJ2vu!4@j58{NN;78i?6__xR75QEDC4JN{>RmgcNrn-EOpEOcyR<8FS@RB@ zH!R7J=`KK^u06eeI|X@}KvQmdKE3AmAy8 zM4IIvde#e4O(iwag`UL5yQo>6&7^=D4yE-Eo9$9R2hR} zn;Z9i-d=R-xZl4@?s%8|m1M`$J6lW1r0Y)+8q$}Vn4qyR1jqTjGH;@Z!2KiGun2~x zaiEfzVT<|_b6t}~XPeflAm8hvCHP3Bp*tl{^y_e{Jsn@w+KP{7}bH_s=1S2E1sj=18a39*Ag~lbkT^_OQuYQey=b zW^{0xlQ@O$^cSxUZ8l(Mspg8z0cL*?yH4;X2}TdN)uN31A%$3$a=4;{S@h#Y(~i%) zc=K7Ggl=&2hYVic*W65gpSPE70pU;FN@3k?BYdNDKv6wlsBAF^);qiqI zhklsX4TaWiC%VbnZ|yqL+Pcc;(#&E*{+Rx&<&R{uTYCn^OD|mAk4%Q7gbbgMnZwE{ zy7QMK%jIjU@ye?0; z;0--&xVeD}m_hq9A8a}c9WkI2YKj8t!Mkk!o%AQ?|CCBL9}n570}OmZ(w)YI6#QS&p<={tcek*D{CPR%eVA1WBGUXf z%gO2vL7iVDr1$!LAW)1@H>GoIl=&yyZ7=*9;wrOYQ}O}u>h}4FWL?N2ivURlUi11- zl{G0fo`9?$iAEN<4kxa#9e0SZPqa{pw?K=tdN5tRc7HDX-~Ta6_+#s9W&d`6PB7dF*G@|!Mc}i zc=9&T+edI(@la}QU2An#wlkJ&7RmTEMhyC_A8hWM54?s1WldCFuBmT5*I3K9=1aj= z6V@93P-lUou`xmB!ATp0(We$?)p*oQs;(Kku15~q9`-LSl{(Efm&@%(zj?aK2;5}P z{6<@-3^k^5FCDT@Z%XABEcuPoumYkiD&)-8z2Q}HO9OVEU3WM;V^$5r4q>h^m73XF z5!hZ7SCjfxDcXyj(({vg8FU(m2_}36L_yR>fnW)u=`1t@mPa76`2@%8v@2@$N@TE` z)kYhGY1jD;B9V=Dv1>BZhR9IJmB?X9Wj99f@MvJ2Fim*R`rsRilvz_3n!nPFLmj({EP!@CGkY5R*Y_dSO{qto~WerlG}DMw9k+n}pk z*nL~7R2gB{_9=zpqX|*vkU-dx)(j+83uvYGP?K{hr*j2pQsfXn<_As6z%-z+wFLqI zMhTkG>2M}#BLIOZ(ya1y8#W<+uUo@(43=^4@?CX{-hAuaJki(_A(uXD(>`lzuM~M;3XA48ZEN@HRV{1nvt?CV)t;|*dow0Ue2`B*iA&!rI`fZQ=b28= z_dxF}iUQ8}nq0SA4NK@^EQ%=)OY;3fC<$goJ&Kp|APQ@qVbS-MtJQBc)^aO8mYFsbhafeRKdHPW&s^&;%>v zlTz`YE}CuQ@_X&mqm{+{!h2r)fPGeM_Ge4RRYQkrma`&G<>RW<>S(?#LJ}O-t)d$< zf}b0svP^Zu@)MqwEV^Fb_j zPYYs~vmEC~cOIE6Nc^@b@nyL!w5o?nQ!$mGq(Pa|1-MD}K0si<&}eag=}WLSDO zE4+eA~!J(K}605x&4 zT72P7J^)Y)b(3g2MZ@1bv%o1ggwU4Yb!DhQ=uu-;vX+Ix8>#y6wgNKuobvrPNx?$3 zI{BbX<=Y-cBtvY&#MpGTgOLYU4W+csqWZx!=AVMb)Z;8%#1*x_(-)teF>45TCRwi1 z)Nn>hy3_lo44n-4A@=L2gI$yXCK0lPmMuldhLxR8aI;VrHIS{Dk}yp= zwjhB6v@0DN=Hnm~3t>`CtnPzvA*Kumfn5OLg&-m&fObRD};c}Hf?n&mS< z%$wztc%kjWjCf-?+q(bZh9k~(gs?i4`XVfqMXvPVkUWfm4+EBF(nOkg!}4u)6I)JT zU6IXqQk?p1a2(bz^S;6ZH3Wy9!JvbiSr7%c$#G1eK2^=~z1WX+VW)CPD#G~)13~pX zErO(>x$J_4qu-)lNlZkLj2}y$OiKn0ad5Imu5p-2dnt)(YI|b7rJ3TBUQ8FB8=&ym50*ibd2NAbj z;JA&hJ$AJlldM+tO;Yl3rBOFiP8fDdF?t(`gkRpmT9inR@uX{bThYNmxx-LN5K8h0 ztS%w*;V%b`%;-NARbNXn9he&AO4$rvmkB#;aaOx?Wk|yBCmN{oMTK&E)`s&APR<-5 z#;_e75z;LJ)gBG~h<^`SGmw<$Z3p`KG|I@7Pd)sTJnouZ1hRvm3}V+#lPGk4b&A#Y z4VSNi8(R1z7-t=L^%;*;iMTIAjrXl;h106hFrR{n9o8vlz?+*a1P{rEZ2ie{luQs} zr6t746>eoqiO5)^y;4H%2~&FT*Qc*9_oC2$+&syHWsA=rn3B~4#QEW zf4GT3i_@)f(Fj}gAZj`7205M8!B&HhmbgyZB& z+COyAVNxql#DwfP;H48Yc+Y~ChV6b9auLnfXXvpjr<~lQ@>VbCpQvWz=lyVf1??_c zAo3C^otZD@(v?X)UX*@w?TF|F8KF>l7%!Dzu+hksSA^akEkx8QD(V(lK+HBCw6C}2onVExW)f$ zncm*HI(_H;jF@)6eu}Tln!t?ynRkcqBA5MitIM@L^(4_Ke}vy7c%$w{(`&7Rn=u>oDM+Z^RUYcbSOPwT(ONyq76R>$V6_M_UP4vs=__I#io{{((| zy5=k=oVr-Qt$FImP~+&sN8rf2UH*vRMpwohPc@9?id17La4weIfBNa>1Djy+1=ugn z@}Zs;eFY1OC}WBDxDF=i=On_33(jWE-QYV)HbQ^VM!n>Ci9_W0Zofz7!m>do@KH;S z4k}FqEAU2)b%B_B-QcPnM5Zh=dQ+4|DJoJwo?)f2nWBuZE@^>a(gP~ObzMuyNJTgJFUPcH`%9UFA(P23iaKgo0)CI!SZ>35LpFaD7 z)C2sW$ltSEYNW%%j8F;yK{iHI2Q^}coF@LX`=EvxZb*_O;2Z0Z5 z7 zlccxmCfCI;_^awp|G748%Wx%?t9Sh8!V9Y(9$B?9R`G)Nd&snX1j+VpuQ@GGk=y(W zK|<$O`Cad`Y4#W3GKXgs%lZduAd1t1<7LwG4*zaStE*S)XXPFDyKdgiaVXG2)LvDn zf}eQ_S(&2!H0Mq1Yt&WpM1!7b#yt_ie7naOfX129_E=)beKj|p1VW9q>>+e$3@G$K zrB%i_TT1DHjOf7IQ8)Wu4#K%ZSCDGMP7Ab|Kvjq7*~@ewPm~h_-8d4jmNH<&mNZC@CI zKxG5O08|@<4(6IEC@L-lcrrvix&_Dj4tBvl=8A}2UX|)~v#V$L22U}UHk`B-1MF(t zU6aVJWR!>Y0@4m0UA%Sq9B5;4hZvsOu=>L`IU4#3r_t}os|vSDVMA??h>QJ1FD1vR z*@rclvfD!Iqoxh>VP+?b9TVH8g@KjYR@rRWQy44A`f6doIi+8VTP~pa%`(Oa@5?=h z8>YxNvA##a3D0)^P|2|+0~f|UsAJV=q(S>eq-dehQ+T>*Q@qN zU8@kdpU5gGk%ozt?%c8oM6neA?GuSsOfU_b1U)uiEP8eRn~>M$p*R z43nSZs@^ahO78s zulbK@@{3=2=@^yZ)DuIC$ki;`2WNbD_#`LOHN9iMsrgzt-T<8aeh z(oXrqI$Kgt6)Icu=?11NWs>{)_ed1wh>)wv6RYNUA-C&bejw{cBE_5Wzeo!AHdTd+ z)d(_IKN7z^n|As~3XS=cCB_TgM7rK;X586re`{~Foml$aKs zb!4Pe7hEP|370EWwn$HKPM!kL94UPZ1%8B^e5fB+=Iw^6=?5n3tZGYjov83CLB&OQ++p)WCMeshCv_9-~G9C_2x`LxTDjUcW$l6e!6-&a^fM3oP9*g(H zmCk0nGt1UMdU#pfg1G0um5|sc|KO<+qU1E4iBF~RvN*+`7uNHH^gu{?nw2DSCjig% zI@ymKZSK=PhHJa(jW&xeApv&JcfSmNJ4uQ|pY=Lcc>=J|{>5Ug3@x#R_b@55xFgfs za^ANzWdD$ZYtFs$d7+oiw0ZmPk2&l|< zc8()wfiJx@EGpQT zG$8iLkQZ-086doF1R zh<#9cz_vRsJdoXbD=QgOtpm}cFAJX8c}>Jew;PQJSXSb^;wlC zxXLHTS|!GZ-VK_4wV<9bk4RUmlsByzW_^b>)$6R+jQ}^wco1nMA`9Lncs;&QGp!`5Tx#aXXU?}5_RrtUY zx(EMzDhl-a^y^f5yfFLMnOO#u)l69&4M?|ne|2EV>zQ}4JQCBel?~2I4?D|>L$%H(peOOII!U}i z-j)*h1rODe9{0`xmhG;`AKqw1p0_KhEIU8)DoGnEn9wAhXPaxO_(jNSij~J5m$P*$ z9Mt(t;eV}2+i|kjQpBFcNb7_(VbuF<;RQB~R~p>2*Lg>a&7DEEuq*I%Ls4{zHeUDq z+M0&YhEn^C*9-B4Q7HJ$xj)dORCXPK+)ZtLOa0o&)Sl+f(Y{p*68$-#yagW5^HQnQ z0pWpoQpxg8<&gx9im(>=x6v#&RbQ7^AsjxeSDA? zi4MEJUC~ByG!PiBjq7$pK&FA^5 z=Y@dtQnuy%IfsaR`TVP0q^3mixl&J-3!$H!ua#{A>0Z1JdLq#d4UV9nlYm641ZHl zH6mK~iI6lR3OUEVL}Z5{ONZ_6{Nk%Bv03ag<1HVN?R%w2^aR5@E>6(r>}IoMl$wRF zWr-DItN*k7T$NTT8B)+23c?171sADhjInb2Xb>GhFYGC&3{b>huvLlaS4O z^{j5q+b5H?Z)yuy%AByaVl2yj9cnalY1sMQ zXI#e%*CLajxGxP!K6xf9RD2pMHOfAa1d^Lr6kE`IBpxOiGXfNcoQ*FI6wsNtLD!T+ zC4r2q>5qz0f}UY^RY#1^0*FPO*Zp-U1h9U|qWjwqJaDB(pZ`<`U-xo7+JB$zvwV}^ z2>$0&Q5k#l|Er7*PPG1ycj4BGz zg&`d*?nUi1Q!OB>{V@T$A;)8@h;*Rb1{xk_8X<34L`s}xkH-rQZvjM`jI=jaJRGRg zeEcjYChf-78|RLrao%4HyZBfnAx5KaE~@Sx+o-2MLJ>j-6uDb!U`odj*=)0k)K75l zo^)8-iz{_k7-_qy{Ko~N#B`n@o#A22YbKiA>0f3k=p-B~XX=`Ug>jl$e7>I=hph0&AK z?ya;(NaKY_!od=tFUcGU5Kwt!c9EPUQLi;JDCT*{90O@Wc>b| zI;&GIY$JlQW^9?R$-OEUG|3sp+hn+TL(YK?S@ZW<4PQa}=IcUAn_wW3d!r#$B}n08 z*&lf(YN21NDJ74DqwV`l`RX(4zJ<(E4D}N0@QaE-hnfdPDku~@yhb^AeZL73RgovX z6=e>!`&e^l@1WA5h!}}PwwL*Gjg!LbC5g0|qb8H$^S{eGs%cc?4vTyVFW=s6KtfW? z@&Xm+E(uz(qDbwDvRQI9DdB<2sW}FYK9sg*f%-i*>*n{t-_wXvg~N7gM|a91B!x|K zyLbJ~6!!JZpZ`#HpCB8g#Q*~VU47Rp$NyZb3WhEgg3ivSwnjGJgi0BEV?!H}Z@QF| zrO`Kx*52;FR#J-V-;`oR-pr!t>bYf)UYcixN=(FUR6$fhN@~i09^3WeP3*)D*`*mJ z1u%klAbzQ=P4s%|FnVTZv%|@(HDB+ap5S#cFSJUSGkyI*Y>9Lwx|0lTs%uhoCW(f1 zi+|a9;vDPfh3nS<7m~wqTM6+pEm(&z-Ll;lFH!w#(Uk#2>Iv~2Hu}lITn7hnOny`~ z*Vj=r<&Nwpq^@g5m`u&QTBRoK*}plAuHg$L$~NO#wF0!*r0OfcS%)k0A??uY*@B^C zJe9WdU(w){rTIf<;rwJt^_35^d<A@$FqEZW6kwyfAo2x0T$Ye2MZox6Z7<%Qbu$}}u{rtE+h2M+Z}T4I zxF1cwJ(Uvp!T#mogWkhb(?SxD4_#tV(Sc8N4Gu*{Fh#})Pvb^ef%jrlnG*&Ie+J5 zsly5oo?1((um&lLDxn(DkYtk`My>lgKTp3Y4?hTQ4_`YNOFtjF-FUY#d#(EQd(rfz zB8z%Vi;?x)ZM$3c>yc5H8KBvSevnWNdCbAj?QCac)6-K~Xz@EZp}~N9q)5*Ufjz3C z6kkOeI{3H(^VO8hKDrVjy2DXd;5wr4nb`19yJi0DO@607MSx+7F$ zz3F7sl8JV@@sM$6`#JmSilqI%Bs)}Py2eFT;TjcG5?8$zwV60b(_5A>b#uk~7U^bO z>y|6SCrP2IGST(8HFuX|XQUXPLt2gL_hm|uj1Ws`O2VW>SyL^uXkl>Zvkcpi?@!F7 z%svLoT@{R#XrIh^*dE~$YhMwC+b7JE09NAS47kT%Ew zD!XjxA@1+KOAyu`H2z#h+pGm!lG>WI0v745l+Fd><3dh{ATq%h?JSdEt zu%J*zfFUx%Tx&0DS5WSbE)vwZSoAGT=;W#(DoiL($BcK;U*w`xA&kheyMLI673HCb7fGkp{_vdV2uo;vSoAH z9BuLM#Vzwt#rJH>58=KXa#O;*)_N{$>l7`umacQ0g$pI3iW4=L--O;Wiq0zy7OKp`j2r^y3`7X!?sq9rr5B{41BkBr1fEd1#Q3 z-dXc2RSb4U>FvpVhlQCIzQ-hs=8420z=7F2F(^xD;^RXgpjlh8S6*xCP#Gj2+Q0bAg?XARw3dnlQ*Lz3vk}m`HXmCgN=?bIL{T zi}Ds-xn|P)dxhraT@XY$ZQ&^%x8y!o+?n#+>+dZ1c{hYwNTNRke@3enT(a@}V*X{! z81+{Jc2UR;+Zcbc6cUlafh4DFKwp>;M}8SGD+YnW3Q_)*9Z_pny_z+MeYQmz?r%EVaN0d!NE*FVPq&U@vo{ef6wkMIDEWLbDs zz91$($XbGnQ?4WHjB~4xgPgKZts{p|g1B{-4##}#c5aL5C6_RJ_(*5>85B1}U!_<``}q-97Q7~u)(&lsb(WT^(*n7H%33%@_b zO5(?-v??s??33b19xiB7t_YT!q8!qAzN1#RD@3;kYAli%kazt#YN7}MhVu=ljuz27 z1`<+g8oVwy57&$`CiHeaM)tz(OSt4E# zJ@P6E*e504oUw~RD(=9WP8QdW^6wRdFbKII!GAWecJ(?{`EzTR@?j!3g?$@LLCt;U={>!9z7DU!(1Jq zqEwdx5q?W1Ncm7mXP8MFwAr?nw5$H%cb>Q><9j{Tk2RY9ngGvaJgWXx^r!ywk{ph- zs2PFto4@IIwBh{oXe;yMZJYlS?3%a-CJ#js90hoh5W5d^OMwCFmpryHFr|mG+*ZP$ zqyS5BW@s}|3xUO0PR<^{a2M(gkP5BDGxvkWkPudSV*TMRK5Qm4?~VuqVAOerffRt$HGAvp;M++Iq$E6alB z;ykBr-eZ6v_H^1Wip56Czj&=`mb^TsX|FPN#-gnlP03AkiJDM=?y|LzER1M93R4sC z*HT(;EV=*F*>!+Z{r!KG?6ODMGvkt3viG=@kQJHNMYd}bS4KrrHf4`&*(0m0R5Hqz zEk)r=sFeS?MZRvn<@Z0&bDw)XkMnw+_xqgp=W{;ioX`6;G-P9N%wfoYJ$-m$L#MC% z^sH?tSzA|WWP(cN3({~_*X$l{M*;1V{l$;T6b){#l4pswDTid26HaXgKed}13YIP= zJRvA3nmx{}R$Lr&S4!kWU3`~dxM}>VXWu6Xd(VP}z1->h&f%82eXD_TuTs@=c;l0T z|LHmWKJ+?7hkY=YM>t}zvb4|lV;!ARMtWFp!E^J=Asu9w&kVF*i{T#}sY++-qnVh! z5TQ|=>)+vutf{&qB+LO9^jm#rD7E5+tcorr^Fn5Xb0B;)f^$7Ev#}G_`r==ea294V z--v4LwjswWlSq9ba6i?IXr8M_VEGQ$H%hCqJTFQ3+1B9tmxDUhnNU%dy4+zbqYJ|o z3!N{b?A@{;cG2~nb-`|z;gEDL5ffF@oc3`R{fGi)0wtMqEkw4tRX3t;LVS3-zAmg^ zgL7Z{hmdPSz9oA@t>tZ1<|Khn&Lp=_!Q=@a?k+t~H&3jN?dr(}7s;{L+jiKY57?WsFBfW^mu6a03_^VKrdK=9egXw@!nzZ3TbYc*osyQNoCXPYoFS<&Nr97MrQCOK(gO8 z;0@iqRTJy4-RH)PJld5`AJN}n?5r^-enKrHQOR;z>UMfm+e8~4ZL5k>oXMiYq12Bx4eVQv0jFgp_zC#``sjZpywYqISMP}VZ@!~1Mf$!x|opj%mQ98JnSk@`~ zPmmyuPZKtZOnEC!1y!?`TYRsZ!II;d!iln}%e}bk5qIiUADERr*K$3dekgHV9TtBX zi5q!J!6Zgd#cLxRmZN^J`o@Zv{+p+<_#8^nvY)44Hw_2i@?R&5n^q33fpOnDg1nPQ z_r<$hURl~OketX|Tdbvf_7=3x^rSFJtEp@tuDpVB&uq)qW;xUQ7mmkr-@eZwa$l+? zoKk``Vz@TH#>jMce*8>@FZ+@BEUdYa_K0i|{*;j9MW3K%pnM*T;@>|o@lMhgLrpZP5aol(z>g;b4}|e$U~Fn zGL%(}p%Jsl4LxE!VW_Y4T>e}W4e#~F03H_^R!Q)kpJG{lO!@I4{mFo^V#ayHh_5~o zB$O71gcE(G@6xv);#Ky?e(Ed}^O+Ho(t=93T9T3TnEY(OVf_dR-gY@jj+iJSY?q|6prBv(S9A4k=2fNZz!W@S=B@~b?TJRTuBQq448@juN#Y=3q=^VCF>Z}n6wICJ<^^Kn8C;mK zZYiFSN#Z$?NDGV7(#}q2tAZAtE63icK-MY>UQu4MWlGIbJ$AF8Zt-jV;@7P5MPI>% zPWvO!t%1+s>-A%`;0^o8Ezeaa4DMwI8ooQrJ;ax@Qt*6XONWw)dPwOPI9@u*EG&844*1~EoZ2qsAe~M>d`;Bc_CWY zMoDKEmDh-}k9d6*<0g@aQmsnrM1H9IcKYZs)><)d92{|0Hh8?~XbF)7U+UmP@Pw_6geVB?7N$4J4*E0z3EO&5kRS(EE zv92(+e5WxLXMN{h;-|8@!Q#0q247hb^3R%*k3MuMO5*L}$0D#5P*N$aHd54C+=_RToYXTyewugOaDmGsCvb4H1s=@gkfVnzTCWKMa-Mm1v4Wq!t-JIrbV&EWwKDe ze#kJpOq#iRlFz%5#6Fio9IUlKnQ#X&DY8Ux#<-WqxAac-y%U_L+EZZ4Rg5*yNg`f< zSZn&uio@zanUCPqX1l4W&B!;UWs#P7B^|4WwoCxQXl|44n^cBNqu=3Vl*ltAqsUQO z9q_@nD0zq0O8r`coEm>9+|rA3HL#l}X;0##>SJS$cVavOZVCpSGf4mUU1( zWaRCUYc^9QbG9=vpWo%xP}CMFnMb{reA`K7tT(t5DM)d9l}jVPY>qoRzT zE3m-p#=i=$9x*CB`AL>SY}u3agYFl#uULNen#&44H;!L@I{RI=PlWxG8J((f)ma7A z@jLvQ>?Nx`n?3ChRG#HqE3MXP8*o3!Qq`+t8EMt_p)oeKHqPusBxPn!#?R??-=e3e zo73WNs_IZF`WLigre=|`aS2^> zN1zn!7k&Dh28t%VpJ%**&E!eAcB5oLjQFFcJQj*URMia%Ya3@q1UQ18=oWMM6`I}iT_&L1gl?*~6nU4q4Z0`H<5yDp(HeZ+RGf9`mM&= zn-qRp%i!g$R;i1d1aMZ{IewNjE@p2+Z{`x{*xL*x$?WV~{BjJpsP&C&JK0HLoyf z`0z^v&fBQSa!I7FU~9MaQ%e|?RP>sM^2PL!mE^Q1Ig_4M$5BRfi72oMYu6Ke?wmDX z@0a%-V|z}b23K=ye(W+fG#w|jJUnT{=KR5jfuq!RX}<1irTDw(${<&}dWQu4;EuE< z@3u4dBkQaCHHM&;cE0z50_V!(vJ1_V)A8?C#eJuLkt!98Z%|Bgzidc0j|z(&o)TCzYlrgZA zC3@i>L!&Gw_~7`>puB97I2lK)lESZQqVXc_8T^G2O#VHhO?IC$g zOYhXJ7)~C<8l|Xrftka@QuowScM{K&0zskoU$Aw~vIRVRF9TEQ4*3=_5)98B`=t8(N%ZuWqmwlW zllAzq=E5_5!sKDXam@w`ZD(nl%LAPxQuEtDcKPqu9LPJvNIITawU#c^PQ2HmZgs)r zH^+gRwZ?0)8IFQgU)+p@0Iqb^tcEoqcB@zhfz_FaOM&_d<|jnU>q5nSKa<@%9|dje zIupcg1!tRiMP4X=oG<7s4|AW&^-Cw4FL9OuI$t zxjc*y;Uw!G7a|jz>E*2+PlR(CemWebS7m-&*CDwnmxbiRqJvQ&os-sC&4OWt^(2@vG4|jui#Df@-D= zh3D%8Y3R6+jRBStSvH9pt&tCI`NK08J1*pC(?OM0h!bS-JK3I}`pDY-fDIaB_*W6KS+TO0Q*%kkeuN6uWITt=TsCGw6uBE710q; zRluI%j{?@jwhM|l5&TB!-TkQs!A=DXRE>u18t@;zndD0M$U@Igrt?UW2; z7%=dsHIVH_LCkGUU0fW&UMjDnvjcc0Mp(mK&;d~ZJ5EJ)#7@aTZvGDFXzFZg2Lq~s z5PR_LazNN)JD5K_uK*Hy{mXuHTkGGv|9V8KP#iQ$3!G*^>7UiE{|1G1A-qg(xH;Xa>&%f|BZkH zG=J^0pHzSAqv5*5ysQ{Puy^-_|IPrii zKS$mE10Zngf>Sgg@BjpRyJbrHeo zD8Ro0LI*W#+9?^xlOS^c>Z^^n^0I|FH^@^`ZR`{H=$ zjO0_$cnpBM7Zcm?H_RXIu-Lu~qweDSV|tEZBZh!e6hQy->}e;d#osZ1hQj{HhHkC0 zJ|F-HKmeTGgDe979ogBz24;@<|I7;TU!IXb@oWMsMECIETmQy`zPtM`|NP}PjzR_u zKMG1Z{%1kWeMfEf(10U#w!clmQ2)JC8zm(Fv!H4dUHQHCFLikID?hrd{0>kCQt?kP zdqn2ZG0}ytcQJ7t_B3s0ZvH3PYjkjQ`Q%;jV@?MK-+z3etBCGGo4f4`y^|AdCs!DH zThTQ;cL5dM{|tB_1y6K3bVa^hx_<9J(}5`2SDz1^0bT!Vm*JV;9~t&{IC{$DUAVV* z{|E=#yN{wNdTY@$6z{_KNA3&%w|vFu1n9XRcM0Ak>`UW!lQ`ah3D4r%}Z diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 744c64d12..cee8b1b9a 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-rc-2-bin.zip networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 79a61d421..1aa94a426 100755 --- a/gradlew +++ b/gradlew @@ -83,10 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,10 +131,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. @@ -144,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -197,11 +198,15 @@ if "$cygwin" || "$msys" ; then done fi -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ diff --git a/gradlew.bat b/gradlew.bat index 93e3f59f1..25da30dbd 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -43,11 +43,11 @@ set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -57,11 +57,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail From 54e3447e76253dfa524cc322079d16c9b5d487b3 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 17:17:11 +0200 Subject: [PATCH 63/80] Update CI definitions --- .github/workflows/nightly.yml | 6 +++--- .github/workflows/test_workflow.yml | 24 +++++++++++++++++------- gradle/configurations.gradle | 6 +++--- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cdd246a4a..10d1dce65 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -27,7 +27,7 @@ jobs: scenarios=$(cat ./artifacts/failures.txt | tr '\n' ',') echo "Failed scenarios: $scenarios" - curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \ - -H 'Content-Type: application/json' \ - -d "{'scenarios': '${scenarios}', 'failed_run_url': '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'}" +# curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \ +# -H 'Content-Type: application/json' \ +# -d "{'scenarios': '${scenarios}', 'failed_run_url': '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'}" diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml index bb74add34..cc961639d 100644 --- a/.github/workflows/test_workflow.yml +++ b/.github/workflows/test_workflow.yml @@ -10,7 +10,7 @@ jobs: test-linux-glibc: strategy: matrix: - java_version: [ 8u362+9, 11.0.18+10, 17.0.6+10, 21.0.1+12 ] + java_version: [ 8u412+9, 11.0.23+12, 17.0.11+12, 21.0.3+12, 22.0.1+12 ] config: ${{ fromJson(inputs.configuration) }} runs-on: ubuntu-latest timeout-minutes: 180 @@ -28,6 +28,8 @@ jobs: find . -type d -name 'jdk*' -maxdepth 1| xargs -I {} mv {} test_jdk - name: Test run: | + sudo sysctl vm.mmap_rnd_bits=28 + set +e export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} @@ -62,7 +64,7 @@ jobs: test-ubuntu-jdk: strategy: matrix: - java_version: [ 11, 17, 21 ] + java_version: [ '11.0.23-librca', '17.0.11-librca', '21.0.3-librca', '22.0.1-librca' ] config: ${{ fromJson(inputs.configuration) }} runs-on: ubuntu-22.04 timeout-minutes: 180 @@ -76,16 +78,20 @@ jobs: - name: Prepare JDK ${{ matrix.java_version }} run: | sudo apt-get update - sudo apt-get install openjdk-${{ matrix.java_version }}-jdk - uname -r + sudo apt-get install -y curl zip unzip + curl -s "https://get.sdkman.io" | bash + source "$HOME/.sdkman/bin/sdkman-init.sh" + echo 'n' | sdk install java ${{ matrix.java_version }} - name: Test run: | + sudo sysctl vm.mmap_rnd_bits=28 + source "$HOME/.sdkman/bin/sdkman-init.sh" + set +e - locate libtsan_preinit.o export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} export LIBC=glibc - export JAVA_TEST_HOME=/usr/lib/jvm/java-${{ matrix.java_version }}-openjdk-amd64 + export JAVA_TEST_HOME=${SDKMAN_DIR}/candidates/java/${{ matrix.java_version }} export JAVA_HOME=$JAVA_HOME export PATH=$JAVA_HOME/bin:$PATH ./gradlew :ddprof-test:test${{ matrix.config }} @@ -137,6 +143,8 @@ jobs: java-version: "11" - name: Test run: | + sudo sysctl vm.mmap_rnd_bits=28 + set +e export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} @@ -183,6 +191,7 @@ jobs: java-version: "11" - name: Prepare JDK ${{ matrix.java_version }} run: | + sudo sysctl vm.mmap_rnd_bits=28 sudo apt-get -y update && sudo apt-get -y install curl set -eux; sudo mkdir -p /usr/lib/jvm/oracle8; @@ -228,7 +237,7 @@ jobs: test-linux-musl: strategy: matrix: - java_version: [ 8u362+9, 11.0.18+10, 17.0.6+10, 21.0.1+12 ] + java_version: [ 8u412+9, 11.0.23+12, 17.0.11+12, 21.0.3+12, 22.0.1+12 ] config: ${{ fromJson(inputs.configuration) }} runs-on: ubuntu-latest timeout-minutes: 180 @@ -292,6 +301,7 @@ jobs: config: ${{ fromJson(inputs.configuration) }} runs-on: ubuntu-latest timeout-minutes: 180 + if: matrix.config == 'release' || matrix.config == 'debug' steps: - uses: actions/checkout@v3 diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index 3d89aa483..6f33ecb85 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -75,11 +75,11 @@ if (ext.libtsan == null) { } def hasAsan() { - return ext.libasan != null + return isMusl() ? false : ext.libasan != null } def hasTsan() { - return ext.libtsan != null + return isMusl() ? false : ext.libtsan != null } def isActive(def name) { @@ -189,7 +189,7 @@ def tsanCompilerArgs = hasTsan() ? [ '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces ] : [] -def tsanLinkerArgs = hasTsan() ? [ +def tsanLinkerArgs = hasTsan() ? [ "-L${file(libtsan).parent}", "-ltsan", "-fsanitize=thread", From eb5d9c389789c1f1ddfb04927b697c840fb3107c Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 17:21:36 +0200 Subject: [PATCH 64/80] Fix vmstruct stackwalker asan issue ScopeDesc may not be able to resolve method object --- ddprof-lib/src/main/cpp/stackWalker.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ddprof-lib/src/main/cpp/stackWalker.cpp b/ddprof-lib/src/main/cpp/stackWalker.cpp index 5dfe5f230..e982f7bcc 100644 --- a/ddprof-lib/src/main/cpp/stackWalker.cpp +++ b/ddprof-lib/src/main/cpp/stackWalker.cpp @@ -279,6 +279,10 @@ int StackWalker::walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, ScopeDesc scope(nm); do { scope_offset = scope.decode(scope_offset); + if (scope.method() == nullptr) { + fillFrame(frames[depth++], BCI_ERROR, "unknown_scope"); + break; + } type = scope_offset > 0 ? FRAME_INLINED : level >= 1 && level <= 3 ? FRAME_C1_COMPILED : FRAME_JIT_COMPILED; fillFrame(frames[depth++], type, scope.bci(), scope.method()->id()); From d0ea979f60f3842915cd53a566c68a077e9b3bb2 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 17:22:04 +0200 Subject: [PATCH 65/80] Do not try to resolve heap usage function on JDK 11 --- ddprof-lib/src/main/cpp/vmStructs.cpp | 29 ++++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ddprof-lib/src/main/cpp/vmStructs.cpp b/ddprof-lib/src/main/cpp/vmStructs.cpp index 6d917dc3a..b3edd49e4 100644 --- a/ddprof-lib/src/main/cpp/vmStructs.cpp +++ b/ddprof-lib/src/main/cpp/vmStructs.cpp @@ -464,20 +464,25 @@ bool VMStructs::isFlagTrue(const char* name) { } const void* VMStructs::findHeapUsageFunc() { - if (_find_flag_func != NULL) { - // The CollectedHeap::memory_usage function is a virtual one - - // G1, Shenandoah and ZGC are overriding it and calling the base class method results - // in asserts triggering. Therefore, we try to locate the concrete overridden method form. - if (isFlagTrue("UseG1GC")) { - return _libjvm->findSymbol("_ZN15G1CollectedHeap12memory_usageEv"); - } else if (isFlagTrue("UseShenandoahGC")) { - return _libjvm->findSymbol("_ZN14ShenandoahHeap12memory_usageEv"); - } else if (isFlagTrue("UseZGC") && VM::java_version() < 21) { - // acessing this method in JDK 21 (generational ZGC) wil cause SIGSEGV - return _libjvm->findSymbol("_ZN14ZCollectedHeap12memory_usageEv"); + if (VM::hotspot_version() < 17) { + // For JDK 11 it is really unreliable to find the memory_usage function - just disable it + return nullptr; + } else { + if (_find_flag_func != NULL) { + // The CollectedHeap::memory_usage function is a virtual one - + // G1, Shenandoah and ZGC are overriding it and calling the base class method results + // in asserts triggering. Therefore, we try to locate the concrete overridden method form. + if (isFlagTrue("UseG1GC")) { + return _libjvm->findSymbol("_ZN15G1CollectedHeap12memory_usageEv"); + } else if (isFlagTrue("UseShenandoahGC")) { + return _libjvm->findSymbol("_ZN14ShenandoahHeap12memory_usageEv"); + } else if (isFlagTrue("UseZGC") && VM::java_version() < 21) { + // acessing this method in JDK 21 (generational ZGC) wil cause SIGSEGV + return _libjvm->findSymbol("_ZN14ZCollectedHeap12memory_usageEv"); + } } + return _libjvm->findSymbol("_ZN13CollectedHeap12memory_usageEv"); } - return _libjvm->findSymbol("_ZN13CollectedHeap12memory_usageEv"); } void VMStructs::initJvmFunctions() { From 2f5c875362113fec413d257c37a4c5ff261c693f Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 18:10:57 +0200 Subject: [PATCH 66/80] Do not attempt to memcpy from a NULL source --- ddprof-lib/src/main/cpp/livenessTracker.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/livenessTracker.cpp b/ddprof-lib/src/main/cpp/livenessTracker.cpp index 1d01aaba6..b6f620c17 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.cpp +++ b/ddprof-lib/src/main/cpp/livenessTracker.cpp @@ -290,7 +290,9 @@ void LivenessTracker::track(JNIEnv* env, AllocEvent &event, jint tid, jobject ob _table[idx].age = 0; _table[idx].frames_size = num_frames; _table[idx].frames = new jvmtiFrameInfo[_table[idx].frames_size]; - memcpy(_table[idx].frames, frames, sizeof(jvmtiFrameInfo) * _table[idx].frames_size); + if (frames != nullptr) { + memcpy(_table[idx].frames, frames, sizeof(jvmtiFrameInfo) * _table[idx].frames_size); + } _table[idx].ctx = Contexts::get(tid); } From e9afed63f37b4cdf1217451a021ab987f10694bb Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 30 May 2024 16:53:02 +0000 Subject: [PATCH 67/80] Remove commented out code --- ddprof-lib/build.gradle | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index f890fd7b2..6518a75aa 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -162,21 +162,7 @@ tasks.whenTaskAdded { task -> } else if (os().isLinux()) { includes "${javaHome()}/include/linux" } - // if (os().isLinux() && archIdentifier() == 'arm64') { - // // aarch64 build is done in clang - // // the clang binding for the non-built-in tasks is somehow messing up system includes - // // so we need to add them manually - // // this ties the build to clang 11 but there is nothing we can do about it - // systemIncludes.from '/usr/include/c++/10' - // systemIncludes.from '/usr/include/aarch64-linux-gnu/c++/10' - // systemIncludes.from '/usr/include/c++/10/backward' - // systemIncludes.from '/usr/local/include' - // systemIncludes.from '/usr/lib/llvm-11/lib/clang/11.0.1/include' - // systemIncludes.from '/usr/include/aarch64-linux-gnu' - // systemIncludes.from '/usr/include' - // } else { systemIncludes.from task.systemIncludes - // } source task.source inputs.files source outputs.dir objectFileDir From 797490274da1c3cba983a30f35e8707e94833a7f Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 31 May 2024 14:05:01 +0200 Subject: [PATCH 68/80] Fix the test job exclusion for musl --- .github/workflows/test_workflow.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml index cc961639d..cbe62d87b 100644 --- a/.github/workflows/test_workflow.yml +++ b/.github/workflows/test_workflow.yml @@ -301,16 +301,21 @@ jobs: config: ${{ fromJson(inputs.configuration) }} runs-on: ubuntu-latest timeout-minutes: 180 - if: matrix.config == 'release' || matrix.config == 'debug' steps: + - name: Set config output + id: set_config + run: echo "::set-output name=config::${{ matrix.config }}" - uses: actions/checkout@v3 + if: steps.set_config.outputs.config == 'release' || steps.set_config.outputs.config == 'debug' - name: Prepare build JDK + if: steps.set_config.outputs.config == 'release' || steps.set_config.outputs.config == 'debug' uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: "11" - name: Prepare JDK ${{ matrix.java_version }} + if: steps.set_config.outputs.config == 'release' || steps.set_config.outputs.config == 'debug' run: | sudo apt-get -y update && sudo apt-get -y install curl g++-9 gcc-9 sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9 @@ -333,6 +338,7 @@ jobs: # rename the bundled libstdc++.so to avoid conflicts with the system one sudo mv /usr/lib/jvm/zing/etc/libc++/libstdc++.so.6 /usr/lib/jvm/zing/etc/libc++/libstdc++.so.6.bak - name: Test + if: steps.set_config.outputs.config == 'release' || steps.set_config.outputs.config == 'debug' run: | set +e export TEST_COMMIT=${{ github.sha }} From ac78bc0fd02befde51487502d1714b9929d6a6c4 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 31 May 2024 14:36:36 +0200 Subject: [PATCH 69/80] Do not attempt to read JavaThread state from non-JavaThread threads (#103) --- ddprof-lib/src/main/cpp/wallClock.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/src/main/cpp/wallClock.cpp b/ddprof-lib/src/main/cpp/wallClock.cpp index 5a9e37bec..3440a3384 100644 --- a/ddprof-lib/src/main/cpp/wallClock.cpp +++ b/ddprof-lib/src/main/cpp/wallClock.cpp @@ -73,7 +73,8 @@ void WallClock::signalHandler(int signo, siginfo_t* siginfo, void* ucontext, u64 ExecutionEvent event; VMThread* vm_thread = VMThread::current(); - int raw_thread_state = vm_thread ? vm_thread->state() : 0; + bool is_java_thread = vm_thread && VM::jni(); + int raw_thread_state = vm_thread && is_java_thread ? vm_thread->state() : 0; bool is_initialized = raw_thread_state >= 4 && raw_thread_state < 12; ThreadState state = ThreadState::UNKNOWN; ExecutionMode mode = ExecutionMode::UNKNOWN; @@ -82,7 +83,7 @@ void WallClock::signalHandler(int signo, siginfo_t* siginfo, void* ucontext, u64 if (os_state != ThreadState::UNKNOWN) { state = os_state; } - mode = VM::jni() != NULL ? convertJvmExecutionState(raw_thread_state) : ExecutionMode::JVM; + mode = is_java_thread ? convertJvmExecutionState(raw_thread_state) : ExecutionMode::JVM; } if (state == ThreadState::UNKNOWN) { if (inSyscall(ucontext)) { From 9653303e1f5f822687a2963faae52d1d03597d2e Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 31 May 2024 18:10:46 +0200 Subject: [PATCH 70/80] Remove asan exclusion for VMThread::state() --- ddprof-lib/src/main/cpp/vmStructs.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/ddprof-lib/src/main/cpp/vmStructs.h b/ddprof-lib/src/main/cpp/vmStructs.h index 3eb3fc8e6..c6b17e0a1 100644 --- a/ddprof-lib/src/main/cpp/vmStructs.h +++ b/ddprof-lib/src/main/cpp/vmStructs.h @@ -444,9 +444,6 @@ class VMThread : VMStructs { return *(int*)(osthread + _osthread_id_offset); } - // This triggers in the test suite due to the absence of robust thread filters. - // We will address this as a separate ticket. - __attribute__((no_sanitize("address"))) int state() { return _thread_state_offset >= 0 ? *(int*) at(_thread_state_offset) : 0; } From fee54ccd0181df8db65204f471991f60e9dd9d1e Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 3 Jun 2024 18:24:43 +0200 Subject: [PATCH 71/80] Add sanity check for elfsymbol value offset (#104) --- ddprof-lib/src/main/cpp/codeCache.h | 4 + ddprof-lib/src/main/cpp/symbols_linux.cpp | 192 ++-------------------- ddprof-lib/src/main/cpp/symbols_linux.h | 178 ++++++++++++++++++++ ddprof-lib/src/test/cpp/elfparser_ut.cpp | 28 ++++ ddprof-lib/src/test/resources/readme.txt | 3 + gradle/configurations.gradle | 4 +- 6 files changed, 230 insertions(+), 179 deletions(-) create mode 100644 ddprof-lib/src/main/cpp/symbols_linux.h create mode 100644 ddprof-lib/src/test/cpp/elfparser_ut.cpp create mode 100644 ddprof-lib/src/test/resources/readme.txt diff --git a/ddprof-lib/src/main/cpp/codeCache.h b/ddprof-lib/src/main/cpp/codeCache.h index 53db44ecc..55f401e0a 100644 --- a/ddprof-lib/src/main/cpp/codeCache.h +++ b/ddprof-lib/src/main/cpp/codeCache.h @@ -189,6 +189,10 @@ class CodeCache { long long memoryUsage() { return _capacity * sizeof(CodeBlob*) + _count * sizeof(NativeFunc); } + + int count() { + return _count; + } }; diff --git a/ddprof-lib/src/main/cpp/symbols_linux.cpp b/ddprof-lib/src/main/cpp/symbols_linux.cpp index e90bf8d1e..319c36a91 100644 --- a/ddprof-lib/src/main/cpp/symbols_linux.cpp +++ b/ddprof-lib/src/main/cpp/symbols_linux.cpp @@ -29,180 +29,11 @@ #include #include #include "symbols.h" +#include "symbols_linux.h" #include "dwarf.h" #include "log.h" #include "safeAccess.h" - -class SymbolDesc { - private: - const char* _addr; - const char* _desc; - - public: - SymbolDesc(const char* s) { - _addr = s; - _desc = strchr(_addr, ' '); - } - - const char* addr() { return (const char*)strtoul(_addr, NULL, 16); } - char type() { return _desc != NULL ? _desc[1] : 0; } - const char* name() { return _desc + 3; } -}; - -class MemoryMapDesc { - private: - const char* _addr; - const char* _end; - const char* _perm; - const char* _offs; - const char* _dev; - const char* _inode; - const char* _file; - - public: - MemoryMapDesc(const char* s) { - _addr = s; - _end = strchr(_addr, '-') + 1; - _perm = strchr(_end, ' ') + 1; - _offs = strchr(_perm, ' ') + 1; - _dev = strchr(_offs, ' ') + 1; - _inode = strchr(_dev, ' ') + 1; - _file = strchr(_inode, ' '); - - if (_file != NULL) { - while (*_file == ' ') _file++; - } - } - - const char* file() { return _file; } - bool isReadable() { return _perm[0] == 'r'; } - bool isExecutable() { return _perm[2] == 'x'; } - const char* addr() { return (const char*)strtoul(_addr, NULL, 16); } - const char* end() { return (const char*)strtoul(_end, NULL, 16); } - unsigned long offs() { return strtoul(_offs, NULL, 16); } - unsigned long inode() { return strtoul(_inode, NULL, 10); } - - unsigned long dev() { - char* colon; - unsigned long major = strtoul(_dev, &colon, 16); - unsigned long minor = strtoul(colon + 1, NULL, 16); - return major << 8 | minor; - } -}; - - -#ifdef __LP64__ -const unsigned char ELFCLASS_SUPPORTED = ELFCLASS64; -typedef Elf64_Ehdr ElfHeader; -typedef Elf64_Shdr ElfSection; -typedef Elf64_Phdr ElfProgramHeader; -typedef Elf64_Nhdr ElfNote; -typedef Elf64_Sym ElfSymbol; -typedef Elf64_Rel ElfRelocation; -typedef Elf64_Dyn ElfDyn; -#define ELF_R_TYPE ELF64_R_TYPE -#define ELF_R_SYM ELF64_R_SYM -#else -const unsigned char ELFCLASS_SUPPORTED = ELFCLASS32; -typedef Elf32_Ehdr ElfHeader; -typedef Elf32_Shdr ElfSection; -typedef Elf32_Phdr ElfProgramHeader; -typedef Elf32_Nhdr ElfNote; -typedef Elf32_Sym ElfSymbol; -typedef Elf32_Rel ElfRelocation; -typedef Elf32_Dyn ElfDyn; -#define ELF_R_TYPE ELF32_R_TYPE -#define ELF_R_SYM ELF32_R_SYM -#endif // __LP64__ - -#if defined(__x86_64__) -# define R_GLOB_DAT R_X86_64_GLOB_DAT -#elif defined(__i386__) -# define R_GLOB_DAT R_386_GLOB_DAT -#elif defined(__arm__) || defined(__thumb__) -# define R_GLOB_DAT R_ARM_GLOB_DAT -#elif defined(__aarch64__) -# define R_GLOB_DAT R_AARCH64_GLOB_DAT -#elif defined(__PPC64__) -# define R_GLOB_DAT R_PPC64_GLOB_DAT -#else -# error "Compiling on unsupported arch" -#endif - -#ifdef __musl__ -static const bool MUSL = true; -#else -static const bool MUSL = false; -#endif // __musl__ - -class ElfParser { - private: - CodeCache* _cc; - const char* _base; - const char* _file_name; - bool _relocate_dyn; - ElfHeader* _header; - const char* _sections; - const char* _vaddr_diff; - - ElfParser(CodeCache* cc, const char* base, const void* addr, const char* file_name, bool relocate_dyn) { - _cc = cc; - _base = base; - _file_name = file_name; - _relocate_dyn = relocate_dyn; - _header = (ElfHeader*)addr; - _sections = (const char*)addr + _header->e_shoff; - } - - bool validHeader() { - unsigned char* ident = _header->e_ident; - return ident[0] == 0x7f && ident[1] == 'E' && ident[2] == 'L' && ident[3] == 'F' - && ident[4] == ELFCLASS_SUPPORTED && ident[5] == ELFDATA2LSB && ident[6] == EV_CURRENT - && _header->e_shstrndx != SHN_UNDEF; - } - - ElfSection* section(int index) { - return (ElfSection*)(_sections + index * _header->e_shentsize); - } - - const char* at(ElfSection* section) { - return (const char*)_header + section->sh_offset; - } - - const char* at(ElfProgramHeader* pheader) { - return _header->e_type == ET_EXEC ? (const char*)pheader->p_vaddr : _vaddr_diff + pheader->p_vaddr; - } - - char* dyn_ptr(ElfDyn* dyn) { - // GNU dynamic linker relocates pointers in the dynamic section, while musl doesn't. - // Also, [vdso] is not relocated, and its vaddr may differ from the load address. - if (_relocate_dyn || (char*)dyn->d_un.d_ptr < _base) { - return (char*)_vaddr_diff + dyn->d_un.d_ptr; - } else { - return (char*)dyn->d_un.d_ptr; - } - } - - ElfSection* findSection(uint32_t type, const char* name); - ElfProgramHeader* findProgramHeader(uint32_t type); - - void calcVirtualLoadAddress(); - void parseDynamicSection(); - void parseDwarfInfo(); - uint32_t getSymbolCount(uint32_t* gnu_hash); - void loadSymbols(bool use_debug); - bool loadSymbolsUsingBuildId(); - bool loadSymbolsUsingDebugLink(); - void loadSymbolTable(const char* symbols, size_t total_size, size_t ent_size, const char* strings); - void addRelocationSymbols(ElfSection* reltab, const char* plt); - - public: - static void parseProgramHeaders(CodeCache* cc, const char* base, const char* end, bool relocate_dyn); - static bool parseFile(CodeCache* cc, const char* base, const char* file_name, bool use_debug); -}; - - ElfSection* ElfParser::findSection(uint32_t type, const char* name) { const char* strtab = at(section(_header->e_shstrndx)); @@ -245,14 +76,16 @@ bool ElfParser::parseFile(CodeCache* cc, const char* base, const char* file_name } size_t length = (size_t)lseek64(fd, 0, SEEK_END); + fprintf(stdout, "===> Parsing file: %s, length=%lu\n", file_name, length); void* addr = mmap(NULL, length, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); if (addr == MAP_FAILED) { Log::warn("Could not parse symbols from %s: %s", file_name, strerror(errno)); } else { - ElfParser elf(cc, base, addr, file_name, false); + ElfParser elf(cc, base != nullptr ? base : (const char*)addr, addr, file_name, length, false); if (elf.validHeader()) { + fprintf(stdout, "===> Loading symbols for: %s (%lu)\n", file_name, length); elf.loadSymbols(use_debug); } munmap(addr, length); @@ -261,7 +94,7 @@ bool ElfParser::parseFile(CodeCache* cc, const char* base, const char* file_name } void ElfParser::parseProgramHeaders(CodeCache* cc, const char* base, const char* end, bool relocate_dyn) { - ElfParser elf(cc, base, base, NULL, relocate_dyn); + ElfParser elf(cc, base, base, NULL, 0, relocate_dyn); if (elf.validHeader() && base + elf._header->e_phoff < end) { cc->setTextBase(base); elf.calcVirtualLoadAddress(); @@ -500,12 +333,17 @@ bool ElfParser::loadSymbolsUsingDebugLink() { } void ElfParser::loadSymbolTable(const char* symbols, size_t total_size, size_t ent_size, const char* strings) { + fprintf(stdout, "===> Loading symbol table at %p, base=%p, size=%lu, entry_size=%lu\n", symbols, _base, total_size, ent_size); for (const char* symbols_end = symbols + total_size; symbols < symbols_end; symbols += ent_size) { ElfSymbol* sym = (ElfSymbol*)symbols; if (sym->st_name != 0 && sym->st_value != 0) { - // Skip special AArch64 mapping symbols: $x and $d - if (sym->st_size != 0 || sym->st_info != 0 || strings[sym->st_name] != '$') { - _cc->add(_base + sym->st_value, (int)sym->st_size, strings + sym->st_name); + if (_length == 0 || (sym->st_name < _length && sym->st_value < _length)) { + // Skip special AArch64 mapping symbols: $x and $d + if (sym->st_size != 0 || sym->st_info != 0 || strings[sym->st_name] != '$') { + _cc->add(_base + sym->st_value, (int)sym->st_size, strings + sym->st_name); + } + } else { + fprintf(stdout, "===> Skipping symbol: value=%lu, name=%u, stt=%d\n", sym->st_value, sym->st_name, sym->st_info & 0xf); } } } @@ -612,7 +450,7 @@ void Symbols::parseLibraries(CodeCacheArray* array, bool kernel_symbols) { while ((len = getline(&str, &str_size, f)) > 0) { str[len - 1] = 0; - + fprintf(stdout, "===> Parsing library: %s\n", str); MemoryMapDesc map(str); if (!map.isReadable() || map.file() == NULL || map.file()[0] == 0) { continue; @@ -647,7 +485,7 @@ void Symbols::parseLibraries(CodeCacheArray* array, bool kernel_symbols) { if (strchr(map.file(), ':') == NULL) { u64 inode = u64(map.dev()) << 32 | map.inode(); if (inode != 0) { - // Do not parse the same executable twice, e.g. on Alpine Linux + // Do not parse the same executable twice, e.g. on Alpine Linux if (_parsed_inodes.insert(inode).second) { if (inode == last_inode) { // If last_inode is set, image_base is known to be valid and readable diff --git a/ddprof-lib/src/main/cpp/symbols_linux.h b/ddprof-lib/src/main/cpp/symbols_linux.h new file mode 100644 index 000000000..057d93f4a --- /dev/null +++ b/ddprof-lib/src/main/cpp/symbols_linux.h @@ -0,0 +1,178 @@ +#ifdef __linux__ +#include "symbols.h" + +#include +#include +#include + +class SymbolDesc { + private: + const char* _addr; + const char* _desc; + + public: + SymbolDesc(const char* s) { + _addr = s; + _desc = strchr(_addr, ' '); + } + + const char* addr() { return (const char*)strtoul(_addr, NULL, 16); } + char type() { return _desc != NULL ? _desc[1] : 0; } + const char* name() { return _desc + 3; } +}; + +class MemoryMapDesc { + private: + const char* _addr; + const char* _end; + const char* _perm; + const char* _offs; + const char* _dev; + const char* _inode; + const char* _file; + + public: + MemoryMapDesc(const char* s) { + _addr = s; + _end = strchr(_addr, '-') + 1; + _perm = strchr(_end, ' ') + 1; + _offs = strchr(_perm, ' ') + 1; + _dev = strchr(_offs, ' ') + 1; + _inode = strchr(_dev, ' ') + 1; + _file = strchr(_inode, ' '); + + if (_file != NULL) { + while (*_file == ' ') _file++; + } + } + + const char* file() { return _file; } + bool isReadable() { return _perm[0] == 'r'; } + bool isExecutable() { return _perm[2] == 'x'; } + const char* addr() { return (const char*)strtoul(_addr, NULL, 16); } + const char* end() { return (const char*)strtoul(_end, NULL, 16); } + unsigned long offs() { return strtoul(_offs, NULL, 16); } + unsigned long inode() { return strtoul(_inode, NULL, 10); } + + unsigned long dev() { + char* colon; + unsigned long major = strtoul(_dev, &colon, 16); + unsigned long minor = strtoul(colon + 1, NULL, 16); + return major << 8 | minor; + } +}; + + +#ifdef __LP64__ +const unsigned char ELFCLASS_SUPPORTED = ELFCLASS64; +typedef Elf64_Ehdr ElfHeader; +typedef Elf64_Shdr ElfSection; +typedef Elf64_Phdr ElfProgramHeader; +typedef Elf64_Nhdr ElfNote; +typedef Elf64_Sym ElfSymbol; +typedef Elf64_Rel ElfRelocation; +typedef Elf64_Dyn ElfDyn; +#define ELF_R_TYPE ELF64_R_TYPE +#define ELF_R_SYM ELF64_R_SYM +#else +const unsigned char ELFCLASS_SUPPORTED = ELFCLASS32; +typedef Elf32_Ehdr ElfHeader; +typedef Elf32_Shdr ElfSection; +typedef Elf32_Phdr ElfProgramHeader; +typedef Elf32_Nhdr ElfNote; +typedef Elf32_Sym ElfSymbol; +typedef Elf32_Rel ElfRelocation; +typedef Elf32_Dyn ElfDyn; +#define ELF_R_TYPE ELF32_R_TYPE +#define ELF_R_SYM ELF32_R_SYM +#endif // __LP64__ + +#if defined(__x86_64__) +# define R_GLOB_DAT R_X86_64_GLOB_DAT +#elif defined(__i386__) +# define R_GLOB_DAT R_386_GLOB_DAT +#elif defined(__arm__) || defined(__thumb__) +# define R_GLOB_DAT R_ARM_GLOB_DAT +#elif defined(__aarch64__) +# define R_GLOB_DAT R_AARCH64_GLOB_DAT +#elif defined(__PPC64__) +# define R_GLOB_DAT R_PPC64_GLOB_DAT +#else +# error "Compiling on unsupported arch" +#endif + +#ifdef __musl__ +static const bool MUSL = true; +#else +static const bool MUSL = false; +#endif // __musl__ + +class ElfParser { + private: + CodeCache* _cc; + const char* _base; + const char* _file_name; + size_t _length; + bool _relocate_dyn; + ElfHeader* _header; + const char* _sections; + const char* _vaddr_diff; + + ElfParser(CodeCache* cc, const char* base, const void* addr, const char* file_name, size_t length, bool relocate_dyn) { + _cc = cc; + _base = base; + _file_name = file_name; + _length = length; + _relocate_dyn = relocate_dyn && base != nullptr; + _header = (ElfHeader*)addr; + _sections = (const char*)addr + _header->e_shoff; + } + + bool validHeader() { + unsigned char* ident = _header->e_ident; + return ident[0] == 0x7f && ident[1] == 'E' && ident[2] == 'L' && ident[3] == 'F' + && ident[4] == ELFCLASS_SUPPORTED && ident[5] == ELFDATA2LSB && ident[6] == EV_CURRENT + && _header->e_shstrndx != SHN_UNDEF; + } + + ElfSection* section(int index) { + return (ElfSection*)(_sections + index * _header->e_shentsize); + } + + const char* at(ElfSection* section) { + return (const char*)_header + section->sh_offset; + } + + const char* at(ElfProgramHeader* pheader) { + return _header->e_type == ET_EXEC ? (const char*)pheader->p_vaddr : _vaddr_diff + pheader->p_vaddr; + } + + char* dyn_ptr(ElfDyn* dyn) { + // GNU dynamic linker relocates pointers in the dynamic section, while musl doesn't. + // Also, [vdso] is not relocated, and its vaddr may differ from the load address. + if (_relocate_dyn || (char*)dyn->d_un.d_ptr < _base) { + return (char*)_vaddr_diff + dyn->d_un.d_ptr; + } else { + return (char*)dyn->d_un.d_ptr; + } + } + + ElfSection* findSection(uint32_t type, const char* name); + ElfProgramHeader* findProgramHeader(uint32_t type); + + void calcVirtualLoadAddress(); + void parseDynamicSection(); + void parseDwarfInfo(); + uint32_t getSymbolCount(uint32_t* gnu_hash); + void loadSymbols(bool use_debug); + bool loadSymbolsUsingBuildId(); + bool loadSymbolsUsingDebugLink(); + void loadSymbolTable(const char* symbols, size_t total_size, size_t ent_size, const char* strings); + void addRelocationSymbols(ElfSection* reltab, const char* plt); + + public: + static void parseProgramHeaders(CodeCache* cc, const char* base, const char* end, bool relocate_dyn); + static bool parseFile(CodeCache* cc, const char* base, const char* file_name, bool use_debug); +}; + +#endif //__linux__ \ No newline at end of file diff --git a/ddprof-lib/src/test/cpp/elfparser_ut.cpp b/ddprof-lib/src/test/cpp/elfparser_ut.cpp new file mode 100644 index 000000000..9db37ce3f --- /dev/null +++ b/ddprof-lib/src/test/cpp/elfparser_ut.cpp @@ -0,0 +1,28 @@ +#ifdef __linux__ + +#include + +#include "codeCache.h" +#include "symbols_linux.h" + +#include +#include // For PATH_MAX + +TEST(Elf, readSymTable) { + char cwd[PATH_MAX - 64]; + if (getcwd(cwd, sizeof(cwd)) == nullptr) { + exit(1); + } + fprintf(stdout, "Current working directory: %s\n", cwd); + char path[PATH_MAX]; + snprintf(path, sizeof(path) - 1, "%s/../src/test/resources/libj9jit.so", cwd); + if (access(path, R_OK) != 0) { + fprintf(stdout, "Missing test resource %s. Skipping the test\n", path); + exit(0); + } + CodeCache cc("test"); + ElfParser::parseFile(&cc, nullptr, path, false); + fprintf(stdout, "CodeCache size: %d\n", cc.count()); +} + +#endif //__linux__ \ No newline at end of file diff --git a/ddprof-lib/src/test/resources/readme.txt b/ddprof-lib/src/test/resources/readme.txt new file mode 100644 index 000000000..ee2a498d1 --- /dev/null +++ b/ddprof-lib/src/test/resources/readme.txt @@ -0,0 +1,3 @@ +TODO! +Create a library exhibiting the problems with the st_value offsets in the ELF file format. +Put it here and update the elfparser_ut.cpp test to use it. diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index 6f33ecb85..271d0db10 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -203,7 +203,7 @@ def tsanEnv = hasTsan() ? // Linux addBuildConfiguration 'release', 'linux', 'x64', - ['-O3', '-DNDEBUG'] + commonLinuxCompilerArgs, + ['-O3', '-DNDEBUG', '-g'] + commonLinuxCompilerArgs, commonLinuxLinkerArgs + [ "-Wl,-z,nodelete", "-static-libstdc++", @@ -226,7 +226,7 @@ addBuildConfiguration 'tsan', 'linux', 'x64', tsanEnv addBuildConfiguration 'release', 'linux', 'arm64', - ['-O3', '-DNDEBUG'] + commonLinuxCompilerArgs, + ['-O3', '-DNDEBUG', '-g'] + commonLinuxCompilerArgs, commonLinuxLinkerArgs + [ "-Wl,-z,nodelete", "-static-libstdc++", From 9e6c37981b539ad0d4157ca0a2cf76a6367e746e Mon Sep 17 00:00:00 2001 From: r1viollet Date: Tue, 4 Jun 2024 10:06:50 +0200 Subject: [PATCH 72/80] Add a binary for unresolved functions --- ddprof-lib/src/test/resources/readme.txt | 3 --- .../test/resources/unresolved-functions/Makefile | 3 +++ .../test/resources/unresolved-functions/linker.ld | 13 +++++++++++++ .../src/test/resources/unresolved-functions/main.c | 10 ++++++++++ .../test/resources/unresolved-functions/readme.txt | 4 ++++ 5 files changed, 30 insertions(+), 3 deletions(-) delete mode 100644 ddprof-lib/src/test/resources/readme.txt create mode 100644 ddprof-lib/src/test/resources/unresolved-functions/Makefile create mode 100644 ddprof-lib/src/test/resources/unresolved-functions/linker.ld create mode 100644 ddprof-lib/src/test/resources/unresolved-functions/main.c create mode 100644 ddprof-lib/src/test/resources/unresolved-functions/readme.txt diff --git a/ddprof-lib/src/test/resources/readme.txt b/ddprof-lib/src/test/resources/readme.txt deleted file mode 100644 index ee2a498d1..000000000 --- a/ddprof-lib/src/test/resources/readme.txt +++ /dev/null @@ -1,3 +0,0 @@ -TODO! -Create a library exhibiting the problems with the st_value offsets in the ELF file format. -Put it here and update the elfparser_ut.cpp test to use it. diff --git a/ddprof-lib/src/test/resources/unresolved-functions/Makefile b/ddprof-lib/src/test/resources/unresolved-functions/Makefile new file mode 100644 index 000000000..d1590d491 --- /dev/null +++ b/ddprof-lib/src/test/resources/unresolved-functions/Makefile @@ -0,0 +1,3 @@ +all: + gcc -c main.c -o main.o + gcc -o main main.o -T linker.ld \ No newline at end of file diff --git a/ddprof-lib/src/test/resources/unresolved-functions/linker.ld b/ddprof-lib/src/test/resources/unresolved-functions/linker.ld new file mode 100644 index 000000000..6f0701d73 --- /dev/null +++ b/ddprof-lib/src/test/resources/unresolved-functions/linker.ld @@ -0,0 +1,13 @@ +SECTIONS +{ + . = 0x10000; + .text : { *(.text) } + . = 0x20000; + .data : { *(.data) } + .bss : { *(.bss) } + . = 0x30000; + unresolved_symbol = .; + . = 0xffffffffffffffff; + unresolved_function = .; + +} diff --git a/ddprof-lib/src/test/resources/unresolved-functions/main.c b/ddprof-lib/src/test/resources/unresolved-functions/main.c new file mode 100644 index 000000000..e55838b2e --- /dev/null +++ b/ddprof-lib/src/test/resources/unresolved-functions/main.c @@ -0,0 +1,10 @@ +#include + +extern int unresolved_symbol; +extern int unresolved_function(); + +int main() { + printf("Value of unresolved_symbol: %p\n", &unresolved_symbol); + printf("Value of unresolved_function: %p\n", &unresolved_function); + return 0; +} diff --git a/ddprof-lib/src/test/resources/unresolved-functions/readme.txt b/ddprof-lib/src/test/resources/unresolved-functions/readme.txt new file mode 100644 index 000000000..bbfe23baf --- /dev/null +++ b/ddprof-lib/src/test/resources/unresolved-functions/readme.txt @@ -0,0 +1,4 @@ +# Description + +This binary tests that we are able to parse symbols even when they point to unresolved functions. +The function is set to point to a 0xffffffffffffffff address. From 38159dcf592f4c882328ec05ffcba372eb9f9a5f Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 4 Jun 2024 11:14:08 +0200 Subject: [PATCH 73/80] Remove debug output --- ddprof-lib/src/main/cpp/symbols_linux.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/ddprof-lib/src/main/cpp/symbols_linux.cpp b/ddprof-lib/src/main/cpp/symbols_linux.cpp index 319c36a91..7c2ea3080 100644 --- a/ddprof-lib/src/main/cpp/symbols_linux.cpp +++ b/ddprof-lib/src/main/cpp/symbols_linux.cpp @@ -76,7 +76,6 @@ bool ElfParser::parseFile(CodeCache* cc, const char* base, const char* file_name } size_t length = (size_t)lseek64(fd, 0, SEEK_END); - fprintf(stdout, "===> Parsing file: %s, length=%lu\n", file_name, length); void* addr = mmap(NULL, length, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); @@ -450,7 +449,6 @@ void Symbols::parseLibraries(CodeCacheArray* array, bool kernel_symbols) { while ((len = getline(&str, &str_size, f)) > 0) { str[len - 1] = 0; - fprintf(stdout, "===> Parsing library: %s\n", str); MemoryMapDesc map(str); if (!map.isReadable() || map.file() == NULL || map.file()[0] == 0) { continue; From fb5015d3d7ba55f563e98cff172175038e05cb14 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Tue, 4 Jun 2024 11:24:03 +0200 Subject: [PATCH 74/80] Tsan fixes - Adjust atomic operations - spinlock - relaxed load - ctimer - avoid possible leak in timers --- ddprof-lib/gtest/build.gradle | 1 + ddprof-lib/src/main/cpp/ctimer_linux.cpp | 6 ++++-- ddprof-lib/src/main/cpp/spinLock.h | 6 ++++-- ddprof-lib/src/main/cpp/symbols_linux.cpp | 5 ----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ddprof-lib/gtest/build.gradle b/ddprof-lib/gtest/build.gradle index b38a6db90..d24f04904 100644 --- a/ddprof-lib/gtest/build.gradle +++ b/ddprof-lib/gtest/build.gradle @@ -7,6 +7,7 @@ if (rootDir.toString().endsWith("ddprof-lib/gradle")) { apply from: rootProject.file('../../common.gradle') } + // disable the default compile and link tasks not to interfere with our custom ones tasks.withType(CppCompile).configureEach { task -> if (task.name.startsWith('compileRelease') || task.name.startsWith('compileDebug')) { diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 77de3056c..1f6124bbe 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -120,8 +120,10 @@ void CTimer::unregisterThread(int tid) { if (tid >= _max_timers) { return; } - - int timer = _timers[tid]; + // Atomic acquire to avoid possible leak when unregistering + // This was raised by tsan, with registers and unregisters done in separate + // threads. + int timer = __atomic_load_n(&_timers[tid], __ATOMIC_ACQUIRE); if (timer != 0 && __sync_bool_compare_and_swap(&_timers[tid], timer--, 0)) { syscall(__NR_timer_delete, timer); } diff --git a/ddprof-lib/src/main/cpp/spinLock.h b/ddprof-lib/src/main/cpp/spinLock.h index e5d7a81ac..e8bacec9b 100644 --- a/ddprof-lib/src/main/cpp/spinLock.h +++ b/ddprof-lib/src/main/cpp/spinLock.h @@ -53,7 +53,8 @@ class SpinLock { bool tryLockShared() { int value; - while ((value = __atomic_load_n(&_lock, __ATOMIC_ACQUIRE)) <= 0) { + // we use relaxed as the compare already offers the guarantees we need + while ((value = __atomic_load_n(&_lock, __ATOMIC_RELAXED)) <= 0) { if (__sync_bool_compare_and_swap(&_lock, value, value - 1)) { return true; } @@ -63,7 +64,8 @@ class SpinLock { void lockShared() { int value; - while ((value = _lock) > 0 || !__sync_bool_compare_and_swap(&_lock, value, value - 1)) { + while ((value = __atomic_load_n(&_lock, __ATOMIC_RELAXED)) > 0 + || !__sync_bool_compare_and_swap(&_lock, value, value - 1)) { spinPause(); } } diff --git a/ddprof-lib/src/main/cpp/symbols_linux.cpp b/ddprof-lib/src/main/cpp/symbols_linux.cpp index 319c36a91..86e630b93 100644 --- a/ddprof-lib/src/main/cpp/symbols_linux.cpp +++ b/ddprof-lib/src/main/cpp/symbols_linux.cpp @@ -76,7 +76,6 @@ bool ElfParser::parseFile(CodeCache* cc, const char* base, const char* file_name } size_t length = (size_t)lseek64(fd, 0, SEEK_END); - fprintf(stdout, "===> Parsing file: %s, length=%lu\n", file_name, length); void* addr = mmap(NULL, length, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); @@ -85,7 +84,6 @@ bool ElfParser::parseFile(CodeCache* cc, const char* base, const char* file_name } else { ElfParser elf(cc, base != nullptr ? base : (const char*)addr, addr, file_name, length, false); if (elf.validHeader()) { - fprintf(stdout, "===> Loading symbols for: %s (%lu)\n", file_name, length); elf.loadSymbols(use_debug); } munmap(addr, length); @@ -342,8 +340,6 @@ void ElfParser::loadSymbolTable(const char* symbols, size_t total_size, size_t e if (sym->st_size != 0 || sym->st_info != 0 || strings[sym->st_name] != '$') { _cc->add(_base + sym->st_value, (int)sym->st_size, strings + sym->st_name); } - } else { - fprintf(stdout, "===> Skipping symbol: value=%lu, name=%u, stt=%d\n", sym->st_value, sym->st_name, sym->st_info & 0xf); } } } @@ -450,7 +446,6 @@ void Symbols::parseLibraries(CodeCacheArray* array, bool kernel_symbols) { while ((len = getline(&str, &str_size, f)) > 0) { str[len - 1] = 0; - fprintf(stdout, "===> Parsing library: %s\n", str); MemoryMapDesc map(str); if (!map.isReadable() || map.file() == NULL || map.file()[0] == 0) { continue; From 98032a2336bdc8c97ff3e28dee43339e81f1999d Mon Sep 17 00:00:00 2001 From: r1viollet Date: Tue, 4 Jun 2024 12:12:44 +0200 Subject: [PATCH 75/80] TSan fixes - Change the enabled flags to atomics --- ddprof-lib/src/main/cpp/ctimer.h | 6 +++--- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 4 ++-- ddprof-lib/src/main/cpp/wallClock.cpp | 4 ++-- ddprof-lib/src/main/cpp/wallClock.h | 5 +++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer.h b/ddprof-lib/src/main/cpp/ctimer.h index e51ed68b4..fe163d606 100644 --- a/ddprof-lib/src/main/cpp/ctimer.h +++ b/ddprof-lib/src/main/cpp/ctimer.h @@ -18,7 +18,7 @@ #define _CTIMER_H #include "engine.h" - +#include #ifdef __linux__ #include @@ -26,7 +26,7 @@ class CTimer : public Engine { private: - static volatile bool _enabled; + static std::atomic _enabled; static long _interval; static CStack _cstack; static int _signal; @@ -58,7 +58,7 @@ class CTimer : public Engine { void stop(); inline void enableEvents(bool enabled) { - _enabled = enabled; + _enabled.store(enabled, std::memory_order_seq_cst); } }; diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 1f6124bbe..0a226a8a7 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -79,7 +79,7 @@ long CTimer::_interval; int CTimer::_max_timers = 0; int* CTimer::_timers = NULL; CStack CTimer::_cstack; -volatile bool CTimer::_enabled = false; +std::atomic CTimer::_enabled{false}; int CTimer::_signal; int CTimer::registerThread(int tid) { @@ -191,7 +191,7 @@ void CTimer::signalHandler(int signo, siginfo_t* siginfo, void* ucontext) { // Save the current errno value int saved_errno = errno; - if (!_enabled) return; + if (!_enabled.load(std::memory_order_relaxed)) return; int tid = 0; ProfiledThread* current = ProfiledThread::current(); if (current != NULL) { diff --git a/ddprof-lib/src/main/cpp/wallClock.cpp b/ddprof-lib/src/main/cpp/wallClock.cpp index 3440a3384..511951dc1 100644 --- a/ddprof-lib/src/main/cpp/wallClock.cpp +++ b/ddprof-lib/src/main/cpp/wallClock.cpp @@ -26,7 +26,7 @@ #include "tsc.h" #include "vmStructs.h" -volatile bool WallClock::_enabled = false; +std::atomic WallClock::_enabled{false}; bool WallClock::inSyscall(void *ucontext) { StackFrame frame(ucontext); @@ -136,7 +136,7 @@ void WallClock::stop() { } void WallClock::timerLoop() { - if (!_enabled) { + if (!_enabled.load(std::memory_order_relaxed)) { return; } std::vector tids; diff --git a/ddprof-lib/src/main/cpp/wallClock.h b/ddprof-lib/src/main/cpp/wallClock.h index 81263a069..9250f5a15 100644 --- a/ddprof-lib/src/main/cpp/wallClock.h +++ b/ddprof-lib/src/main/cpp/wallClock.h @@ -27,7 +27,8 @@ class WallClock : public Engine { private: - static volatile bool _enabled; + static std::atomic _enabled; + bool _collapsing; long _interval; @@ -76,7 +77,7 @@ class WallClock : public Engine { void stop(); inline void enableEvents(bool enabled) { - _enabled = enabled; + _enabled.store(enabled, std::memory_order_seq_cst); } }; From 706b40ba47630cc5a23f55be48d415603630a7ad Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 4 Jun 2024 12:28:56 +0200 Subject: [PATCH 76/80] Plug the test reources build into gradle scripts (#105) --- ddprof-lib/gtest/build.gradle | 20 +++++++++++ ddprof-lib/src/main/cpp/symbols_linux.cpp | 2 +- ddprof-lib/src/test/cpp/elfparser_ut.cpp | 3 +- .../resources/unresolved-functions/Makefile | 5 +-- .../resources/unresolved-functions/linker.ld | 36 +++++++++++++++++-- 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/ddprof-lib/gtest/build.gradle b/ddprof-lib/gtest/build.gradle index d24f04904..e1b83c35a 100644 --- a/ddprof-lib/gtest/build.gradle +++ b/ddprof-lib/gtest/build.gradle @@ -37,6 +37,22 @@ tasks.withType(StripSymbols).configureEach { task -> } } +def buildResourcesTask = tasks.register("buildResources", Exec) { + group = 'build' + description = "Build the resources for the Google Tests" + + onlyIf { + hasGtest && !project.hasProperty('skip-native') && os().isLinux() + } + + def targetDir = project(':ddprof-lib').file('build/test/resources/unresolved-functions') + + commandLine "sh", "-c", "cd ${project(':ddprof-lib').projectDir}/src/test/resources/unresolved-functions && make TARGET_DIR=${targetDir}" + + inputs.files project(':ddprof-lib').files('src/test/resources/unresolved-functions') + outputs.file "${targetDir}/main" +} + def gtestAll = tasks.register("gtest") { onlyIf { hasGtest && !project.hasProperty('skip-native') @@ -156,6 +172,10 @@ tasks.whenTaskAdded { task -> gtestLinkTask.dependsOn compileTask } gtestTask.get().dependsOn gtestExecuteTask.get() + if (os().isLinux()) { + // custom binaries for tests are built only on linux + gtestExecuteTask.get().dependsOn buildResourcesTask + } gtestAll.get().dependsOn gtestExecuteTask.get() } } diff --git a/ddprof-lib/src/main/cpp/symbols_linux.cpp b/ddprof-lib/src/main/cpp/symbols_linux.cpp index 86e630b93..b3ea40452 100644 --- a/ddprof-lib/src/main/cpp/symbols_linux.cpp +++ b/ddprof-lib/src/main/cpp/symbols_linux.cpp @@ -331,10 +331,10 @@ bool ElfParser::loadSymbolsUsingDebugLink() { } void ElfParser::loadSymbolTable(const char* symbols, size_t total_size, size_t ent_size, const char* strings) { - fprintf(stdout, "===> Loading symbol table at %p, base=%p, size=%lu, entry_size=%lu\n", symbols, _base, total_size, ent_size); for (const char* symbols_end = symbols + total_size; symbols < symbols_end; symbols += ent_size) { ElfSymbol* sym = (ElfSymbol*)symbols; if (sym->st_name != 0 && sym->st_value != 0) { + // sanity check the offsets not to exceed the file size if (_length == 0 || (sym->st_name < _length && sym->st_value < _length)) { // Skip special AArch64 mapping symbols: $x and $d if (sym->st_size != 0 || sym->st_info != 0 || strings[sym->st_name] != '$') { diff --git a/ddprof-lib/src/test/cpp/elfparser_ut.cpp b/ddprof-lib/src/test/cpp/elfparser_ut.cpp index 9db37ce3f..6bff98fa9 100644 --- a/ddprof-lib/src/test/cpp/elfparser_ut.cpp +++ b/ddprof-lib/src/test/cpp/elfparser_ut.cpp @@ -13,9 +13,8 @@ TEST(Elf, readSymTable) { if (getcwd(cwd, sizeof(cwd)) == nullptr) { exit(1); } - fprintf(stdout, "Current working directory: %s\n", cwd); char path[PATH_MAX]; - snprintf(path, sizeof(path) - 1, "%s/../src/test/resources/libj9jit.so", cwd); + snprintf(path, sizeof(path) - 1, "%s/../build/test/resources/unresolved-functions/main", cwd); if (access(path, R_OK) != 0) { fprintf(stdout, "Missing test resource %s. Skipping the test\n", path); exit(0); diff --git a/ddprof-lib/src/test/resources/unresolved-functions/Makefile b/ddprof-lib/src/test/resources/unresolved-functions/Makefile index d1590d491..2f3f66f1f 100644 --- a/ddprof-lib/src/test/resources/unresolved-functions/Makefile +++ b/ddprof-lib/src/test/resources/unresolved-functions/Makefile @@ -1,3 +1,4 @@ +TARGET_DIR = ../build/test/resources/unresolved-functions all: - gcc -c main.c -o main.o - gcc -o main main.o -T linker.ld \ No newline at end of file + gcc -c main.c -o $(TARGET_DIR)/main.o + gcc -o $(TARGET_DIR)/main $(TARGET_DIR)/main.o -T linker.ld \ No newline at end of file diff --git a/ddprof-lib/src/test/resources/unresolved-functions/linker.ld b/ddprof-lib/src/test/resources/unresolved-functions/linker.ld index 6f0701d73..0e930a398 100644 --- a/ddprof-lib/src/test/resources/unresolved-functions/linker.ld +++ b/ddprof-lib/src/test/resources/unresolved-functions/linker.ld @@ -1,13 +1,43 @@ +PHDRS +{ + headers PT_PHDR PHDRS ; + interp PT_INTERP ; + text PT_LOAD FILEHDR PHDRS ; + data PT_LOAD ; +} + SECTIONS { . = 0x10000; - .text : { *(.text) } + .text : { + *(.text) + } :text + . = 0x20000; - .data : { *(.data) } - .bss : { *(.bss) } + .data : { + *(.data) + } :data + + .bss : { + *(.bss) + } + . = 0x30000; unresolved_symbol = .; . = 0xffffffffffffffff; unresolved_function = .; + /* Add the .init_array section */ + .init_array : { + __init_array_start = .; + KEEP(*(.init_array)) + __init_array_end = .; + } + + /* Add the .fini_array section */ + .fini_array : { + __fini_array_start = .; + KEEP(*(.fini_array)) + __fini_array_end = .; + } } From c9cc5b0198b7db243f5e9b343424de049edb3bea Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 4 Jun 2024 12:41:08 +0200 Subject: [PATCH 77/80] Try setting up images with gtest --- .github/workflows/test_workflow.yml | 34 ++++++++++++++++++++-------- README.md | 3 +-- ddprof-lib/build.gradle | 2 -- ddprof-lib/gtest/build.gradle | 5 +++- ddprof-lib/src/main/cpp/dictionary.h | 1 + gradle/configurations.gradle | 3 --- 6 files changed, 31 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml index cbe62d87b..589567448 100644 --- a/.github/workflows/test_workflow.yml +++ b/.github/workflows/test_workflow.yml @@ -21,6 +21,10 @@ jobs: with: distribution: 'temurin' java-version: "11" + - name: Setup OS + run: | + sudo apt-get update + sudo apt-get install -y curl zip unzip libgtest-dev libgmock-dev - name: Prepare JDK ${{ matrix.java_version }} run: | wget -nv https://download.bell-sw.com/java/${{ matrix.java_version }}/bellsoft-jdk${{ matrix.java_version }}-linux-amd64.tar.gz -O jdk.tar.gz @@ -66,7 +70,7 @@ jobs: matrix: java_version: [ '11.0.23-librca', '17.0.11-librca', '21.0.3-librca', '22.0.1-librca' ] config: ${{ fromJson(inputs.configuration) }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 180 steps: - uses: actions/checkout@v3 @@ -75,10 +79,12 @@ jobs: with: distribution: 'temurin' java-version: "11" - - name: Prepare JDK ${{ matrix.java_version }} + - name: Prepare OS run: | sudo apt-get update - sudo apt-get install -y curl zip unzip + sudo apt-get install -y curl zip unzip libgtest-dev libgmock-dev + - name: Prepare JDK ${{ matrix.java_version }} + run: | curl -s "https://get.sdkman.io" | bash source "$HOME/.sdkman/bin/sdkman-init.sh" echo 'n' | sdk install java ${{ matrix.java_version }} @@ -129,6 +135,11 @@ jobs: timeout-minutes: 180 steps: - uses: actions/checkout@v3 + - name: Setup OS + run: | + sudo apt-get update + sudo apt-get install -y curl zip unzip libgtest-dev libgmock-dev +# sudo sysctl vm.mmap_rnd_bits=28 - name: Prepare test JDK uses: actions/setup-java@v3 with: @@ -143,8 +154,6 @@ jobs: java-version: "11" - name: Test run: | - sudo sysctl vm.mmap_rnd_bits=28 - set +e export TEST_COMMIT=${{ github.sha }} export TEST_CONFIGURATION=glibc/${{ matrix.java_version }} @@ -180,10 +189,14 @@ jobs: strategy: matrix: config: ${{ fromJson(inputs.configuration) }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 180 steps: - uses: actions/checkout@v3 + - name: Setup OS + run: | + sudo apt-get update + sudo apt-get install -y curl zip unzip libgtest-dev libgmock-dev - name: Prepare build JDK uses: actions/setup-java@v3 with: @@ -192,7 +205,6 @@ jobs: - name: Prepare JDK ${{ matrix.java_version }} run: | sudo sysctl vm.mmap_rnd_bits=28 - sudo apt-get -y update && sudo apt-get -y install curl set -eux; sudo mkdir -p /usr/lib/jvm/oracle8; # https://gist.github.com/wavezhang/ba8425f24a968ec9b2a8619d7c2d86a6?permalink_comment_id=4444663#gistcomment-4444663 @@ -247,8 +259,8 @@ jobs: options: --cpus 2 steps: - uses: actions/checkout@v3 - - name: Setup system - run: apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake >/dev/null + - name: Setup OS + run: apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake gtest-dev gmock >/dev/null - name: Prepare build JDK run: | wget -nv https://download.bell-sw.com/java/11.0.18+10/bellsoft-jdk11.0.18+10-linux-x64-musl.tar.gz -O jdk.tar.gz @@ -308,6 +320,10 @@ jobs: run: echo "::set-output name=config::${{ matrix.config }}" - uses: actions/checkout@v3 if: steps.set_config.outputs.config == 'release' || steps.set_config.outputs.config == 'debug' + - name: Setup OS + run: | + sudo apt-get update + sudo apt-get install -y curl zip unzip libgtest-dev libgmock-dev - name: Prepare build JDK if: steps.set_config.outputs.config == 'release' || steps.set_config.outputs.config == 'debug' uses: actions/setup-java@v3 diff --git a/README.md b/README.md index 366ffbdb4..ab6e37a18 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,7 @@ If you need a full-fledged Java profiler head back to [async-profiler](https://g 1. This project contains Java code and is built by gradle, as such, you need to install a JDK in order to build it. Run `brew install openjdk` to install it on MacOS. If building and testing on Apple Silicon (M1+), be sure to install an aarch64 OpenJDK distribution, otherwise the architecture used for compilation and the JVM based test suite will not match, which means the tests will not be able to load the dynamic library at runtime. 2. The build requires make, which is included in XCode, but can also be installed by running `brew install make` -3. The build may attempt to use `locate` to validate tsan installation. You might need to install it via `sudo apt install locate` -4. The build supports googletest for unit testing. To install it, run `brew install googletest` or `sudo apt install libgtest-dev` depending on your platform. +4. The build supports googletest for unit testing. To install it, run `brew install googletest` or `sudo apt install libgtest-dev` or `apk add gtest-dev` depending on your platform. Once prerequisites have been installed simple as `./gradlew assembleAll`. The resulting artifact is located in `ddprof-lib/build/libs/ddprof-.jar` diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle index 6518a75aa..aedc1ed5e 100644 --- a/ddprof-lib/build.gradle +++ b/ddprof-lib/build.gradle @@ -309,8 +309,6 @@ tasks.withType(Test) { // relink the tasks when all are created gradle.projectsEvaluated { -// tasks.copyReleaseLibs.dependsOn tasks.stripLibRelease - buildConfigNames().each { def compileTask = tasks.findByName("compileLib${it.capitalize()}") def linkTask = tasks.findByName("linkLib${it.capitalize()}") diff --git a/ddprof-lib/gtest/build.gradle b/ddprof-lib/gtest/build.gradle index e1b83c35a..ef3f0ca5f 100644 --- a/ddprof-lib/gtest/build.gradle +++ b/ddprof-lib/gtest/build.gradle @@ -139,7 +139,10 @@ tasks.whenTaskAdded { task -> description = "Link the Google Test for the ${config.name} build of the library" source = fileTree("$buildDir/obj/gtest/${config.name}/${testName}") linkedFile = binary - linkerArgs.addAll(config.linkerArgs) + if (config.name != 'release') { + // linking the gtests using the minimizing release flags is making gtest unhappy + linkerArgs.addAll(config.linkerArgs) + } linkerArgs.addAll("-lgtest", "-lgtest_main", "-lgmock", "-lgmock_main") if (os().isMacOsX()) { linkerArgs.addAll("-L/opt/homebrew/opt/googletest/lib") diff --git a/ddprof-lib/src/main/cpp/dictionary.h b/ddprof-lib/src/main/cpp/dictionary.h index 1cce3ab59..a12719bae 100644 --- a/ddprof-lib/src/main/cpp/dictionary.h +++ b/ddprof-lib/src/main/cpp/dictionary.h @@ -20,6 +20,7 @@ #include "counters.h" #include #include +#include #define ROW_BITS 7 diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle index 271d0db10..c689a9bb4 100644 --- a/gradle/configurations.gradle +++ b/gradle/configurations.gradle @@ -58,10 +58,7 @@ def locateLibasan() { } def locateLibtsan() { - def libtsan_preinit = locate('libtsan_preinit.o') - if (libtsan_preinit != null && file(libtsan_preinit).exists()) { return locateLibrary('libtsan') - } } ext.libasan = locateLibasan() From 3a4f7e873b687f2c8ed08dbda36983f041a51db3 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Tue, 4 Jun 2024 14:01:17 +0200 Subject: [PATCH 78/80] Adjust memory orders for enabled flag --- ddprof-lib/src/main/cpp/ctimer.h | 2 +- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 4 ++-- ddprof-lib/src/main/cpp/wallClock.cpp | 2 +- ddprof-lib/src/main/cpp/wallClock.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer.h b/ddprof-lib/src/main/cpp/ctimer.h index fe163d606..f67288fda 100644 --- a/ddprof-lib/src/main/cpp/ctimer.h +++ b/ddprof-lib/src/main/cpp/ctimer.h @@ -58,7 +58,7 @@ class CTimer : public Engine { void stop(); inline void enableEvents(bool enabled) { - _enabled.store(enabled, std::memory_order_seq_cst); + _enabled.store(enabled, std::memory_order_release); } }; diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 0a226a8a7..6df9ef133 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -190,8 +190,8 @@ void CTimer::stop() { void CTimer::signalHandler(int signo, siginfo_t* siginfo, void* ucontext) { // Save the current errno value int saved_errno = errno; - - if (!_enabled.load(std::memory_order_relaxed)) return; + // we want to ensure memory order because of the possibility the instance gets cleared + if (!_enabled.load(std::memory_order_acquire)) return; int tid = 0; ProfiledThread* current = ProfiledThread::current(); if (current != NULL) { diff --git a/ddprof-lib/src/main/cpp/wallClock.cpp b/ddprof-lib/src/main/cpp/wallClock.cpp index 511951dc1..b0de35621 100644 --- a/ddprof-lib/src/main/cpp/wallClock.cpp +++ b/ddprof-lib/src/main/cpp/wallClock.cpp @@ -136,7 +136,7 @@ void WallClock::stop() { } void WallClock::timerLoop() { - if (!_enabled.load(std::memory_order_relaxed)) { + if (!_enabled.load(std::memory_order_acquire)) { return; } std::vector tids; diff --git a/ddprof-lib/src/main/cpp/wallClock.h b/ddprof-lib/src/main/cpp/wallClock.h index 9250f5a15..9d0772a55 100644 --- a/ddprof-lib/src/main/cpp/wallClock.h +++ b/ddprof-lib/src/main/cpp/wallClock.h @@ -77,7 +77,7 @@ class WallClock : public Engine { void stop(); inline void enableEvents(bool enabled) { - _enabled.store(enabled, std::memory_order_seq_cst); + _enabled.store(enabled, std::memory_order_release); } }; From 33c8c13c847b1e2fc4f6d97a202a6c33d142bd31 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Thu, 13 Jun 2024 09:23:31 +0200 Subject: [PATCH 79/80] Valgrind analysis - silence an UMR Minor: Silence an UMR in the callTraceStorage as mmap is not correctly instrumented --- ddprof-lib/src/main/cpp/callTraceStorage.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/callTraceStorage.cpp b/ddprof-lib/src/main/cpp/callTraceStorage.cpp index f816e1290..20822d18e 100644 --- a/ddprof-lib/src/main/cpp/callTraceStorage.cpp +++ b/ddprof-lib/src/main/cpp/callTraceStorage.cpp @@ -50,7 +50,9 @@ class LongHashTable { if (table != NULL) { table->_prev = prev; table->_capacity = capacity; - table->_size = 0; + // The reset is not useful with the anon mmap setting the memory is zeroed. + // However this silences a false positive and should not have a performance impact. + table->clear(); } return table; } From be6fbbd37015c37dc429795bc98add0d350fc70d Mon Sep 17 00:00:00 2001 From: r1viollet Date: Tue, 18 Jun 2024 09:54:25 +0200 Subject: [PATCH 80/80] epoch minor atomic adjustement Relax the ordering constraints --- ddprof-lib/src/main/cpp/profiler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 034cf147a..dd72351b4 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -1143,7 +1143,7 @@ Error Profiler::start(Arguments& args, bool reset) { _state = RUNNING; _start_time = time(NULL); - __atomic_add_fetch(&_epoch, 1, __ATOMIC_SEQ_CST); + __atomic_add_fetch(&_epoch, 1, __ATOMIC_RELAXED); return Error::OK; }