Fix use-def analysis for vararg functions.

Reported by Shmuel Zeigerman.

(cherry-picked from commit 0e53a31)

Use-def analysis for BC_FUNCV may consider slots greater than the amount
of non-vararg parameters as dead slots due to the early return (see case
`BCMlit`) for BC_RET emitted before usage of BC_VARG. This patch
restricts the maxslot to be analyzed in such case with the amount of
parameters for the prototype of the current function being recorded.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#8516
Relates to tarantool/tarantool#8718

src/lj_snap.c

@@ -301,8 +301,10 @@ static BCReg snap_usedef(jit_State *J, uint8_t *udf,
 void lj_snap_purge(jit_State *J)
 {
   uint8_t udf[SNAP_USEDEF_SLOTS];
-  BCReg maxslot = J->maxslot;
-  BCReg s = snap_usedef(J, udf, J->pc, maxslot);
+  BCReg s, maxslot = J->maxslot;
+  if (bc_op(*J->pc) == BC_FUNCV && maxslot > J->pt->numparams)
+    maxslot = J->pt->numparams;
+  s = snap_usedef(J, udf, J->pc, maxslot);
   for (; s < maxslot; s++)
     if (udf[s] != 0)
       J->base[s] = 0;  /* Purge dead slots. */

test/tarantool-tests/lj-704-bc-varg-use-def.test.lua

@@ -6,7 +6,7 @@ local test = tap.test('lj-704-bc-varg-use-def'):skipcond({
   ['Test requires JIT enabled'] = not jit.status(),
 })
 
-test:plan(1)
+test:plan(2)
 
 -- XXX: we don't really need to store these builtins, but this is
 -- reduces `jitdump()` output for reader significantly.
@@ -62,4 +62,32 @@ wrap(ON_TRACE_VALUE)
 
 test:ok(result ~= 0, 'use-def analysis for BC_VARG')
 
+-- Now check the same case, but with BC_RET before the BC_VARG,
+-- so use-def analysis will take early return case for BCMlit.
+-- See `snap_usedef()` in <src/lj_snap.c> for details.
+-- The test checks that slots greater than `numparams` are not
+-- purged.
+local function varg_ret_bc(...)
+  -- XXX: This branch contains BC_RET. See the comment above.
+  -- luacheck: ignore
+  if false then else end
+  local slot = ({...})[1]
+  -- Forcify stitch and usage of vararg slot. Any NIY is OK here.
+  return fmod(ARG_ON_RECORDING, slot)
+end
+
+-- XXX: Duplicate wrapper code to avoid recording of `wrap()`
+-- instead of the function to test.
+local function wrap_ret_bc(arg)
+  _, result = pcall(varg_ret_bc, arg)
+end
+
+-- Record trace with the 0 result.
+wrap_ret_bc(ARG_ON_RECORDING)
+wrap_ret_bc(ARG_ON_RECORDING)
+-- Record trace with the non-zero result.
+wrap_ret_bc(ON_TRACE_VALUE)
+
+test:ok(result ~= 0, 'use-def analysis for FUNCV with return before BC_VARG')
+
 os.exit(test:check() and 0 or 1)