Fix frame for more types of on-trace error messages.

Thanks to Maxim Kokryashkin.

(cherry-picked from commit d5bbf9c)

This patch fixes the same issue with frame, as the previous
one, but now for the table overflow error in the `err_msgv`
function. The test for the problem uses the table of GC
finalizers, although they are not required to reproduce the
issue. They only used to make the test as simple as possible.

Resolves tarantool/tarantool#562
Part of tarantool/tarantool#8825

src/lj_err.c

@@ -875,6 +875,10 @@ LJ_NORET LJ_NOINLINE static void err_msgv(lua_State *L, ErrMsg em, ...)
   const char *msg;
   va_list argp;
   va_start(argp, em);
+  if (LJ_HASJIT) {
+    TValue *base = tvref(G(L)->jit_base);
+    if (base) L->base = base;
+  }
   if (curr_funcisL(L)) L->top = curr_topL(L);
   msg = lj_strfmt_pushvf(L, err2msg(em), argp);
   va_end(argp);

test/tarantool-tests/lj-1034-tabov-error-frame.test.lua

@@ -0,0 +1,27 @@
+local tap = require('tap')
+local ffi = require('ffi')
+local test = tap.test('lj-1034-tabov-error-frame'):skipcond({
+  ['Test requires JIT enabled'] = not jit.status(),
+  ['Test requires GC64 mode enabled'] = not ffi.abi('gc64'),
+  ['Disabled on MacOS due to #8652'] = jit.os == 'OSX',
+})
+
+test:plan(2)
+
+-- luacheck: no unused
+local anchor = {}
+local function on_gc(t) end
+
+local function test_finalizers()
+    local i = 1
+    while true do
+        anchor[i] = ffi.gc(ffi.cast('void *', 0), on_gc)
+        i = i + 1
+    end
+end
+
+local st, err = pcall(test_finalizers)
+st, err = pcall(test_finalizers)
+test:ok(st == false, 'error handled successfully')
+test:like(err, '^.+table overflow', 'error is table overflow')
+test:done(true)