Fix frame for more types of on-trace error messages.

Thanks to Maxim Kokryashkin. LuaJIT#1034

(cherry-picked from commit d5bbf9c)

This patch fixes the same issue with frame, as the previous
one, but now for the table overflow error in the `err_msgv`
function. The test for the problem uses the table of GC
finalizers, although they are not required to reproduce the
issue. They only used to make the test as simple as possible.

Resolves tarantool/tarantool#562
Resolves tarantool/tarantool#8652
Part of tarantool/tarantool#8825

CMakeLists.txt

@@ -190,13 +190,6 @@ endif()
 
 # Disable exception unwinding on traces.
 option(LUAJIT_DISABLE_UNWIND_JIT "LuaJIT on-trace exceptions unwinding support" OFF)
-# XXX: On some MacOS versions older than MacOS 13, there is a
-# misbehaviour in libunwind which may potentially lead to an
-# incorrectly unwound stack and segfault. For more info see
-# https://github.com/tarantool/tarantool/issues/8652
-if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
-  set(LUAJIT_DISABLE_UNWIND_JIT ON)
-endif()
 if(LUAJIT_DISABLE_UNWIND_JIT)
   AppendFlags(TARGET_C_FLAGS -DLUAJIT_DISABLE_UNWIND_JIT)
 endif()

src/lj_err.c

@@ -900,6 +900,10 @@ LJ_NORET LJ_NOINLINE static void err_msgv(lua_State *L, ErrMsg em, ...)
   const char *msg;
   va_list argp;
   va_start(argp, em);
+  if (LJ_HASJIT) {
+    TValue *base = tvref(G(L)->jit_base);
+    if (base) L->base = base;
+  }
   if (curr_funcisL(L)) L->top = curr_topL(L);
   msg = lj_strfmt_pushvf(L, err2msg(em), argp);
   va_end(argp);

test/tarantool-tests/gh-7745-oom-on-trace.test.lua

@@ -2,10 +2,6 @@ local tap = require('tap')
 local ffi = require('ffi')
 
 local test = tap.test('OOM on trace'):skipcond({
-  -- luacheck: no global
-  ['Broken unwiding in tarantool_panic_handler'] = _TARANTOOL and
-                                                   (jit.os == 'OSX'),
-  ['Disabled on MacOS due to #8652'] = jit.os == 'OSX',
   ['Test requires JIT enabled'] = not jit.status(),
 })