Fix maxslots when recording BC_VARG, part 2.

Analyzed by Sergey Kaplun.

(cherry-picked from commit a01cba9)

This patch is a follow-up to the previous one. It removes the
condition for `maxslot` changing in the case when varargs are defined
on the trace (i.e. `framedepth` > 0).

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#8825

src/lj_record.c

@@ -1775,12 +1775,8 @@ static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults)
   if (J->framedepth > 0) {  /* Simple case: varargs defined on-trace. */
     ptrdiff_t i;
     if (nvararg < 0) nvararg = 0;
-    if (nresults == -1) {
-      nresults = nvararg;
-      J->maxslot = dst + (BCReg)nvararg;
-    } else if (dst + nresults > J->maxslot) {
-      J->maxslot = dst + (BCReg)nresults;
-    }
+    if (nresults == -1) nresults = nvararg;
+    J->maxslot = dst + (BCReg)nresults;
     for (i = 0; i < nresults; i++)
       J->base[dst+i] = i < nvararg ? getslot(J, i - nvararg - 1 - LJ_FR2) : TREF_NIL;
   } else {  /* Unknown number of varargs passed to trace. */

test/tarantool-tests/lj-1024-varg-maxslot.test.lua

@@ -3,7 +3,7 @@ local test = tap.test('lj-1024-varg-usedef'):skipcond({
   ['Test requires JIT enabled'] = not jit.status(),
 })
 
-test:plan(1)
+test:plan(2)
 
 jit.opt.start('hotloop=1')
 
@@ -32,4 +32,32 @@ end
 
 test:ok(true, 'BC_VARG recording 0th frame depth')
 
+-- Now the same case, but with an additional frame, so VARG slots
+-- are defined on the trace.
+local function varg_frame(...)
+  -- BC_VARG 1 1 0. `...` is nil (argument for the script).
+  -- We have the following bytecodes to be recorded:
+  -- 0001  . . KSHORT   0   1
+  -- 0002  . . KSHORT   1   1
+  -- 0003  . . ISLE     0   1
+  -- 0004  . . JMP      0 => 0007
+  -- 0007  . . KPRI     0   2
+  -- 0008  . . VARG     1   1   0
+  --
+  -- 0002 KSHORT bytecode uses the 2nd JIT slot and the 1st Lua
+  -- slot. This Lua slot will be set to nil after 0008 VARG
+  -- bytecode execution, so after VARG recording maxslot should
+  -- point to the 1st JIT slot.
+  -- luacheck: ignore
+  anchor = 1 >= 1, ...
+end
+
+counter = 0
+while counter < 3 do
+  counter = counter + 1
+  varg_frame()
+end
+
+test:ok(true, 'BC_VARG recording with VARG slots defined on trace')
+
 os.exit(test:check() and 0 or 1)