Futex_fatal_error in ./nptl/pthread_mutex_lock.c

[paintedveil5]

reversion
c3ead3f
master

Build platform
Ubuntu 22.04.2 LTS (Linux 5.19.0-43-generic x86_64)

Build steps

export CXXFLAGS="-g"
export CFLAGS="-g"
./build.sh
./ch poc.js

Test case
poc.js

for(;;);

Backtrace

#0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=140702902117952) at ./nptl/pthread_kill.c:44
#1  __pthread_kill_internal (signo=6, threadid=140702902117952) at ./nptl/pthread_kill.c:78
#2  __GI___pthread_kill (threadid=140702902117952, signo=signo@entry=6) at ./nptl/pthread_kill.c:89
#3  0x00007ffff7242476 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#4  0x00007ffff72287f3 in __GI_abort () at ./stdlib/abort.c:79
#5  0x00007ffff728945c in __libc_message (action=do_abort, fmt=0x7ffff73db7b1 "%s", fmt=0x7ffff73db7b1 "%s", action=do_abort) at ../sysdeps/posix/libc_fatal.c:155
#6  0x00007ffff7289770 in __GI___libc_fatal (message=message@entry=0x7ffff73ddd28 "The futex facility returned an unexpected error code.\n") at ../sysdeps/posix/libc_fatal.c:164
#7  0x00007ffff7291368 in futex_fatal_error () at ../sysdeps/nptl/futex-internal.h:87
#8  futex_wait (private=<optimized out>, expected=2, futex_word=0x7ffff7291368 <__GI___lll_lock_wait+88>) at ../sysdeps/nptl/futex-internal.h:162
#9  __GI___lll_lock_wait (futex=futex@entry=0x55555560823a, private=<optimized out>) at ./nptl/lowlevellock.c:49
#10 0x00007ffff72980dd in lll_mutex_lock_optimized (mutex=0x55555560823a) at ./nptl/pthread_mutex_lock.c:48
#11 ___pthread_mutex_lock (mutex=0x55555560823a) at ./nptl/pthread_mutex_lock.c:128
#12 0x00007ffff4a2248a in AutoCriticalSection::AutoCriticalSection (this=<optimized out>, cs=<optimized out>) at /home/dell/ChakaraCore-debug/lib/Common/Core/CriticalSection.h:61
#13 Js::FunctionProxy::GetAuxPtrWithLock (this=0x7ffff749c000, e=Js::FunctionProxy::AuxPointerType::FormalsPropIdArray) at /home/dell/ChakaraCore-debug/lib/Runtime/Base/FunctionBody.cpp:132
#14 Js::FunctionProxy::GetAuxPtrWithLock<(Js::FunctionProxy::AuxPointerType)21, Js::PropertyIdArray*> (this=0x7ffff749c000) at /home/dell/ChakaraCore-debug/lib/Runtime/./Base/FunctionBody.h:977
#15 Js::FunctionBody::GetFormalsPropIdArray (this=0x7ffff749c000, checkForNull=173) at /home/dell/ChakaraCore-debug/lib/Runtime/Base/FunctionBody.cpp:1052
#16 0x00007ffff4928e89 in JITTimeFunctionBody::InitializeJITFunctionData (arena=0x7ff7f27fdb98, functionBody=0x7ffff749c000, jitBody=0x7ffff74500e0)
    at /home/dell/ChakaraCore-debug/lib/Backend/JITTimeFunctionBody.cpp:211
#17 0x00007ffff4866bef in FunctionJITTimeInfo::BuildJITTimeData (alloc=<optimized out>, codeGenData=0x7ffff785c140, runtimeData=<optimized out>, jitData=0x7ffff7450030, isInlinee=84,
    isForegroundJIT=<optimized out>) at /home/dell/ChakaraCore-debug/lib/Backend/FunctionJITTimeInfo.cpp:41
#18 0x00007ffff497d83c in NativeCodeGenerator::CodeGen (this=this@entry=0x5555555f4020, pageAllocator=pageAllocator@entry=0x555555630450, workItem=workItem@entry=0x555555638f00, foreground=false)
    at /home/dell/ChakaraCore-debug/lib/Backend/NativeCodeGenerator.cpp:981
#19 0x00007ffff497e1cf in NativeCodeGenerator::Process (this=0x5555555f4020, job=<optimized out>, threadData=0x555555630430) at /home/dell/ChakaraCore-debug/lib/Backend/NativeCodeGenerator.cpp:1863
#20 0x00007ffff49bda27 in JsUtil::BackgroundJobProcessor::Process (job=0x555555638f08, threadData=0x555555630430, this=<optimized out>) at /home/dell/ChakaraCore-debug/lib/Common/Common/Jobs.cpp:1037
#21 JsUtil::BackgroundJobProcessor::Run (this=this@entry=0x5555555f5180, threadData=threadData@entry=0x555555630430) at /home/dell/ChakaraCore-debug/lib/Common/Common/Jobs.cpp:1135
#22 0x00007ffff49bc9e7 in JsUtil::BackgroundJobProcessor::StaticThreadProc (lpParam=0x555555630430) at /home/dell/ChakaraCore-debug/lib/Common/Common/Jobs.cpp:1319
#23 0x00007ffff47e10e5 in CorUnix::CPalThread::ThreadEntry (pvParam=0x5555556307c0) at /home/dell/ChakaraCore-debug/pal/src/thread/pal_thread.cpp:1554
#24 0x00007ffff7294b43 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#25 0x00007ffff7326a00 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

[EmployedRussian]

Reproduced for me. Does not reproduce when I change CFLAGS='-g -O0'.

I am seeing the crash stack trace from the original report, and also:

#0  __pthread_kill_implementation (threadid=<optimized out>, signo=6, no_tid=no_tid@entry=0) at ./nptl/pthread_kill.c:44
#1  0x00007ffff74a9d2f in __pthread_kill_internal (signo=<optimized out>, threadid=<optimized out>) at ./nptl/pthread_kill.c:89
#2  __GI___pthread_kill (threadid=<optimized out>, signo=<optimized out>) at ./nptl/pthread_kill.c:89
#3  0x00007ffff75f2e70 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#4  0x00007ffff7445472 in __GI_abort () at ./stdlib/abort.c:79
#5  0x00007ffff749e2d0 in __libc_message (action=action@entry=do_abort, fmt=fmt@entry=0x7ffff75b807e "%s") at ../sysdeps/posix/libc_fatal.c:155
#6  0x00007ffff749e302 in __GI___libc_fatal (message=message@entry=0x7ffff75ba680 "The futex facility returned an unexpected error code.\n") at ../sysdeps/posix/libc_fatal.c:164
#7  0x00007ffff74a4f6a in futex_fatal_error () at ../sysdeps/nptl/futex-internal.h:162
#8  futex_wait (private=<optimized out>, expected=2, futex_word=0x7ffff74a4f6a <__GI___lll_lock_wait+74>) at ../sysdeps/nptl/futex-internal.h:162
#9  __GI___lll_lock_wait (futex=futex@entry=0x5555555f41e1, private=<optimized out>) at ./nptl/lowlevellock.c:49
#10 0x00007ffff74ab2ba in lll_mutex_lock_optimized (mutex=<error reading variable: Cannot access memory at address 0xf33fdd78>) at ./nptl/pthread_mutex_lock.c:48
#11 ___pthread_mutex_lock (mutex=<error reading variable: Cannot access memory at address 0xf33fdd78>) at ./nptl/pthread_mutex_lock.c:128
#12 0x00007ffff4bbeaeb in JsUtil::BackgroundJobProcessor::Run (this=this@entry=0x5555555f41b0, threadData=threadData@entry=0x55555562f3d0) at /h/ChakraCore/lib/Common/Common/Jobs.cpp:1119
#13 0x00007ffff4bbd9f7 in JsUtil::BackgroundJobProcessor::StaticThreadProc (lpParam=0x55555562f3d0) at /h/ChakraCore/lib/Common/Common/Jobs.cpp:1319
#14 0x00007ffff49e20f5 in CorUnix::CPalThread::ThreadEntry (pvParam=0x55555562f760) at /h/ChakraCore/pal/src/thread/pal_thread.cpp:1554
#15 0x00007ffff74a7fd4 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#16 0x00007ffff752866c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

[EmployedRussian]

The problem is caused by this code in pal/inc/cclock.hpp:

3fb33a8271 (rhuanjl       2023-02-14 17:25:04 +0000 18) class CCLOCK_ALIGN CCLock
f13af493bd (Oguz Bastemur 2017-10-30 20:30:13 +0100 19) {
f13af493bd (Oguz Bastemur 2017-10-30 20:30:13 +0100 20)     char           mutexPtr[64]; // keep mutex implementation opaque to consumer (PAL vs non-PAL)
f13af493bd (Oguz Bastemur 2017-10-30 20:30:13 +0100 21)
f13af493bd (Oguz Bastemur 2017-10-30 20:30:13 +0100 22) public:
f13af493bd (Oguz Bastemur 2017-10-30 20:30:13 +0100 23)     void Reset(bool shouldTrackThreadId = false);
f13af493bd (Oguz Bastemur 2017-10-30 20:30:13 +0100 24)     CCLock(bool shouldTrackThreadId = false)
f13af493bd (Oguz Bastemur 2017-10-30 20:30:13 +0100 25)     {
f13af493bd (Oguz Bastemur 2017-10-30 20:30:13 +0100 26)         *((size_t*)mutexPtr) = 0;
f13af493bd (Oguz Bastemur 2017-10-30 20:30:13 +0100 27)         Reset(shouldTrackThreadId);
f13af493bd (Oguz Bastemur 2017-10-30 20:30:13 +0100 28)     }

That does not guarantee that mutexPtr is aligned on any reasonable boundary, and with the compiler on Ubuntu it ends up not being aligned:

(gdb) up
#3  0x0000555555793c96 in Memory::PageAllocatorBase<Memory::VirtualAllocWrapper, Memory::SegmentBase<Memory::VirtualAllocWrapper>, Memory::PageSegmentBase<Memory::VirtualAllocWrapper> >::RequestAlloc (this=0x555556231730, byteCount=131072) at /h/ChakraCore/lib/Common/Memory/PageAllocator.h:987
987                 return policyManager->RequestAlloc(byteCount);
(gdb) p policyManager
$12 = (AllocationPolicyManager *) 0x555556231300

(gdb) p &policyManager.cs
$13 = (CriticalSection *) 0x555556231311   <<<--- not a valid `size_t*` or `pthread_mutex_t*`

[EmployedRussian]

This patch fixes the crash for me:

diff --git a/pal/inc/cclock.hpp b/pal/inc/cclock.hpp
index fe74235a6..1269e9212 100644
--- a/pal/inc/cclock.hpp
+++ b/pal/inc/cclock.hpp
@@ -9,14 +9,9 @@
 #ifndef CC_PAL_INC_CCLOCK_H
 #define CC_PAL_INC_CCLOCK_H

-#if defined(_M_ARM64)
-#define CCLOCK_ALIGN __declspec(align(8))
-#else
-#define CCLOCK_ALIGN
-#endif
-
-class CCLOCK_ALIGN CCLock
+class CCLock
 {
+    __declspec(align(sizeof(size_t)))
     char           mutexPtr[64]; // keep mutex implementation opaque to consumer (PAL vs non-PAL)

 public: