Actually implement maxirconst trace limit.

Suggested by spacewander.

(cherry picked from 0a9ff94)

`maxirconst` should restrict the amount of IR constants per trace.
Nevertheless, its value isn't checked anywhere.

This patch adds the corresponding check after instruction recording.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#6548

src/lj_record.c

@@ -2470,8 +2470,9 @@ void lj_record_ins(jit_State *J)
 #undef rbv
 #undef rcv
 
-  /* Limit the number of recorded IR instructions. */
-  if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord])
+  /* Limit the number of recorded IR instructions and constants. */
+  if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord] ||
+      J->cur.nk < REF_BIAS-(IRRef)J->param[JIT_P_maxirconst])
     lj_trace_err(J, LJ_TRERR_TRACEOV);
 }
 

test/tarantool-tests/lj-430-maxirconst.test.lua

@@ -0,0 +1,43 @@
+-- XXX: avoid any other traces compilation due to hotcount
+-- collisions for predictable results.
+jit.off()
+jit.flush()
+
+-- Disabled on *BSD due to #4819.
+require('utils').skipcond(jit.os == 'BSD', 'Disabled due to #4819')
+
+local tap = require('tap')
+
+local test = tap.test('lj-430-maxirconst')
+test:plan(2)
+
+-- XXX: trace always has at least 3 IR constants: for nil, false
+-- and true.
+jit.opt.start('hotloop=1', 'maxirconst=3')
+
+-- This function has only 3 IR constant.
+local function irconst3()
+end
+
+-- This function has 4 IR constants before optimizations.
+local function irconst4()
+  local _ = 42
+end
+
+local ntrace_old = misc.getmetrics().jit_trace_num
+jit.on()
+irconst3()
+irconst3()
+jit.off()
+test:ok(ntrace_old + 1 == misc.getmetrics().jit_trace_num,
+        'trace number increases')
+
+ntrace_old = misc.getmetrics().jit_trace_num
+jit.on()
+irconst4()
+irconst4()
+jit.off()
+test:ok(ntrace_old == misc.getmetrics().jit_trace_num,
+        'trace number is the same')
+
+os.exit(test:check() and 0 or 1)