Merge pull request #142 from lukego/integrate-dec2017

Integrate multiple feature branches

check-generated-code.nix

@@ -11,8 +11,10 @@ overrideDerivation raptorjit (as:
   {
     checkPhase = ''
       for f in ${generatedFiles}; do
+        echo "checking $f.."
         diff -u src/reusevm/$f src/$f
       done
+      echo "all files ok"
     '';
     doCheck = true;
   })

src/lj_asm.c

@@ -1984,6 +1984,10 @@ void lj_asm_trace(jit_State *J, GCtrace *T)
   as->loopinv = 0;
   as->parent = J->parent ? traceref(J, J->parent) : NULL;
 
+  /* Initialize mcode size of IR instructions array. */
+  T->szirmcode = lj_mem_new(J->L, T->nins * sizeof(*T->szirmcode));
+  memset(T->szirmcode, 0, T->nins * sizeof(*T->szirmcode));
+
   /* Reserve MCode memory. */
   as->mctop = origtop = lj_mcode_reserve(J, &as->mcbot);
   as->mcp = as->mctop;
@@ -2043,6 +2047,7 @@ void lj_asm_trace(jit_State *J, GCtrace *T)
     /* Assemble a trace in linear backwards order. */
     for (as->curins--; as->curins > as->stopins; as->curins--) {
       IRIns *ir = IR(as->curins);
+      MCode *end = as->mcp;
       lua_assert(!(LJ_32 && irt_isint64(ir->t)));  /* Handled by SPLIT. */
       if (!ra_used(ir) && !ir_sideeff(ir) && (as->flags & JIT_F_OPT_DCE))
 	continue;  /* Dead-code elimination can be soooo easy. */
@@ -2051,6 +2056,7 @@ void lj_asm_trace(jit_State *J, GCtrace *T)
       RA_DBG_REF();
       checkmclim(as);
       asm_ir(as, ir);
+      T->szirmcode[as->curins] = (uint16_t)(end - as->mcp);
     }
 
     if (as->realign && J->curfinal->nins >= T->nins)

src/lj_crecord.c

@@ -1289,12 +1289,17 @@ static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
       if (mm == MM_sub) {  /* Pointer difference. */
 	TRef tr;
 	CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
-	if (sz == 0 || (sz & (sz-1)) != 0)
-	  return 0;  /* NYI: integer division. */
-	tr = emitir(IRT(IR_SUB, IRT_INTP), sp[0], sp[1]);
-	tr = emitir(IRT(IR_BSAR, IRT_INTP), tr, lj_ir_kint(J, lj_fls(sz)));
-	tr = emitconv(tr, IRT_NUM, IRT_INTP, 0);
-	return tr;
+        if (sz == 0) {
+          return 0;
+        }
+        tr = emitir(IRT(IR_SUB, IRT_INTP), sp[0], sp[1]);
+        if ((sz & (sz-1)) == 0) { /* special case: divide using bit-shift */
+          tr = emitir(IRT(IR_BSAR, IRT_INTP), tr, lj_ir_kint(J, lj_fls(sz)));
+        } else {                  /* general case: divide using division */
+          tr = emitir(IRT(IR_DIV, IRT_INTP), tr, lj_ir_kint(J, sz));
+        }
+        tr = emitconv(tr, IRT_NUM, IRT_INTP, 0);
+        return tr;
       } else {  /* Pointer comparison (unsigned). */
 	/* Assume true comparison. Fixup and emit pending guard later. */
 	IROp op = mm == MM_eq ? IR_EQ : mm == MM_lt ? IR_ULT : IR_ULE;

src/lj_jit.h

@@ -192,13 +192,16 @@ typedef struct GCtrace {
   MSize szmcode;	/* Size of machine code. */
   MCode *mcode;		/* Start of machine code. */
   MSize mcloop;		/* Offset of loop start in machine code. */
+  uint16_t *szirmcode;  /* Bytes of mcode for each IR instruction (array.) */
   uint16_t nchild;	/* Number of child traces (root trace only). */
   uint16_t spadjust;	/* Stack pointer adjustment (offset in bytes). */
   TraceNo1 traceno;	/* Trace number. */
   TraceNo1 link;	/* Linked trace (or self for loops). */
   TraceNo1 root;	/* Root trace of side trace (or 0 for root traces). */
   TraceNo1 nextroot;	/* Next root trace for same prototype. */
   TraceNo1 nextside;	/* Next side trace of same root trace. */
+  TraceNo1 parent;      /* Parent of this trace (or 0 for root traces). */
+  ExitNo exitno;        /* Exit number in parent (valid for side-traces only). */
   uint8_t sinktags;	/* Trace has SINK tags. */
   uint8_t unused1;
 } GCtrace;
@@ -220,13 +223,13 @@ static LJ_AINLINE MSize snap_nextofs(GCtrace *T, SnapShot *snap)
 /* Round-robin penalty cache for bytecodes leading to aborted traces. */
 typedef struct HotPenalty {
   MRef pc;		/* Starting bytecode PC. */
-  uint16_t val;		/* Penalty value, i.e. hotcount start. */
+  uint32_t val;		/* Penalty value, i.e. hotcount start. */
   uint16_t reason;	/* Abort reason (really TraceErr). */
 } HotPenalty;
 
 #define PENALTY_SLOTS	64	/* Penalty cache slot. Must be a power of 2. */
 #define PENALTY_MIN	(36*2)	/* Minimum penalty value. */
-#define PENALTY_MAX	60000	/* Maximum penalty value. */
+#define PENALTY_MAX	6000000	/* Maximum penalty value. */
 #define PENALTY_RNDBITS	4	/* # of random bits to add to penalty value. */
 
 /* Round-robin backpropagation cache for narrowing conversions. */

src/lj_obj.h

@@ -20,37 +20,36 @@ typedef uint32_t MSize;
 typedef uint64_t GCSize;
 
 /* Memory reference */
-typedef struct MRef {
-  uint64_t ptr64;	/* True 64 bit pointer. */
-} MRef;
+typedef void * MRef;
 
-#define mref(r, t)	((t *)(void *)(r).ptr64)
+#define mref(r, t)	((t *)(r))
 
-#define setmref(r, p)	((r).ptr64 = (uint64_t)(void *)(p))
-#define setmrefr(r, v)	((r).ptr64 = (v).ptr64)
+#define setmref(r, p)	((r) = (void *)(p))
+#define setmrefr(r, v)	((r) = (v))
 
 /* -- GC object references (32 bit address space) ------------------------- */
 
+/* Forward declaration. */
+union GCobj;
+
 /* GCobj reference */
-typedef struct GCRef {
-  uint64_t gcptr64;	/* True 64 bit pointer. */
-} GCRef;
+typedef union GCobj * GCRef;
 
 /* Common GC header for all collectable objects. */
 #define GCHeader	GCRef nextgc; uint8_t marked; uint8_t gct
 /* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */
 
-#define gcref(r)	((GCobj *)(r).gcptr64)
-#define gcrefp(r, t)	((t *)(void *)(r).gcptr64)
-#define gcrefu(r)	((r).gcptr64)
-#define gcrefeq(r1, r2)	((r1).gcptr64 == (r2).gcptr64)
+#define gcref(r)	(r)
+#define gcrefp(r, t)	((t *)(void *)(r))
+#define gcrefu(r)	((uint64_t)(r))
+#define gcrefeq(r1, r2)	((r1)==(r2))
 
-#define setgcref(r, gc)	((r).gcptr64 = (uint64_t)&(gc)->gch)
+#define setgcref(r, gc)	((r) = (GCobj *)&(gc)->gch)
 #define setgcreft(r, gc, it) \
-  (r).gcptr64 = (uint64_t)&(gc)->gch | (((uint64_t)(it)) << 47)
-#define setgcrefp(r, p)	((r).gcptr64 = (uint64_t)(p))
-#define setgcrefnull(r)	((r).gcptr64 = 0)
-#define setgcrefr(r, v)	((r).gcptr64 = (v).gcptr64)
+  (r) = (GCobj *)((uint64_t)&(gc)->gch | ((uint64_t)(it) << 47))
+#define setgcrefp(r, p) ((r) = (GCobj *)p)
+#define setgcrefnull(r)	((r) = NULL)
+#define setgcrefr(r, v)	((r) = (v))
 
 #define gcnext(gc)	(gcref((gc)->gch.nextgc))
 
@@ -438,13 +437,22 @@ typedef struct GCtab {
 
 /* VM states. */
 enum {
+  /* VM states. */
   LJ_VMST_INTERP,	/* Interpreter. */
   LJ_VMST_C,		/* C function. */
   LJ_VMST_GC,		/* Garbage collector. */
   LJ_VMST_EXIT,		/* Trace exit handler. */
   LJ_VMST_RECORD,	/* Trace recorder. */
   LJ_VMST_OPT,		/* Optimizer. */
   LJ_VMST_ASM,		/* Assembler. */
+  /* JIT trace states.
+  ** These are "abstract" states that logically exist but are never
+  ** directly used for the value of global_State.vmstate.
+  */
+  LJ_VMST_HEAD,		/* Trace mcode before loop */
+  LJ_VMST_LOOP,		/* Trace mcode inside loop */
+  LJ_VMST_JGC,		/* GC invoked from JIT mcode. */
+  LJ_VMST_FFI,		/* Other code outside trace mcode */
   LJ_VMST__MAX
 };
 
@@ -518,6 +526,7 @@ typedef struct global_State {
   GCState gc;		/* Garbage collector. */
   volatile int32_t vmstate;   /* VM state or current JIT code trace number. */
   volatile int32_t gcvmstate; /* Previous VM state (only when state is GC). */
+  volatile int32_t lasttrace; /* VM state before exit to interpreter. */
   SBuf tmpbuf;		/* Temporary string buffer. */
   GCstr strempty;	/* Empty string. */
   uint8_t stremptyz;	/* Zero terminator of empty string. */

src/lj_record.c

@@ -568,7 +568,8 @@ static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev)
 /* Handle the case when an already compiled loop op is hit. */
 static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev)
 {
-  if (J->parent == 0 && J->exitno == 0) {  /* Root trace hit an inner loop. */
+  /* Root trace hit an inner loop. */
+  if (J->parent == 0 && J->exitno == 0 && !innerloopleft(J, J->startpc)) {
     /* Better let the inner loop spawn a side trace back here. */
     lj_trace_err(J, LJ_TRERR_LINNER);
   } else if (ev != LOOPEV_LEAVE) {  /* Side trace enters a compiled loop. */
@@ -2291,8 +2292,6 @@ void lj_record_ins(jit_State *J)
   case BC_IFORL:
   case BC_IITERL:
   case BC_ILOOP:
-  case BC_IFUNCF:
-  case BC_IFUNCV:
     lj_trace_err(J, LJ_TRERR_BLACKL);
     break;
 
@@ -2304,13 +2303,15 @@ void lj_record_ins(jit_State *J)
   /* -- Function headers -------------------------------------------------- */
 
   case BC_FUNCF:
+  case BC_IFUNCF:
     rec_func_lua(J);
     break;
   case BC_JFUNCF:
     rec_func_jit(J, rc);
     break;
 
   case BC_FUNCV:
+  case BC_IFUNCV:
     rec_func_vararg(J);
     rec_func_lua(J);
     break;

src/lj_state.c

@@ -171,7 +171,7 @@ static void close_state(lua_State *L)
   lj_mem_free(g, J->bclog, sizeof(BCRecLog)*65536);
   lj_mem_free(g, J->snapmapbuf, J->sizesnapmap);
   lj_mem_free(g, J->snapbuf, J->sizesnap);
-  lj_mem_free(g, J->irbuf-REF_BIAS, 65536*sizeof(IRIns));
+  lj_mem_free(g, J->irbuf, 65536*sizeof(IRIns));
   lua_assert(g->gc.total == sizeof(GG_State));
 #ifndef LUAJIT_USE_SYSMALLOC
   if (g->allocf == lj_alloc_f)
@@ -220,10 +220,10 @@ LUA_API lua_State *lua_newstate(lua_Alloc f, void *ud)
   J->maxbclog = 65536;
   J->bclog = (BCRecLog *)lj_mem_new(L, sizeof(BCRecLog)*J->maxbclog);
   J->nbclog = 0;
-  IRIns *irbufmem = (IRIns *)lj_mem_new(L, sizeof(IRIns)*65536);
-  if (irbufmem == NULL || J->snapbuf == NULL || J->snapmapbuf == NULL)
+  J->irbuf = (IRIns *)lj_mem_new(L, sizeof(IRIns)*65536);
+  if (J->irbuf == NULL || J->snapbuf == NULL ||
+      J->bclog == NULL || J->snapmapbuf == NULL)
     return NULL;
-  J->irbuf = irbufmem + REF_BIAS;
   lj_dispatch_init((GG_State *)L);
   L->status = LUA_ERRERR+1;  /* Avoid touching the stack upon memory error. */
   if (lj_vm_cpcall(L, NULL, NULL, cpluaopen) != 0) {

src/lj_trace.c

@@ -109,6 +109,8 @@ static void trace_save(jit_State *J, GCtrace *T)
   size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
   char *p = (char *)T + sztr;
   memcpy(T, &J->cur, sizeof(GCtrace));
+  T->parent = J->parent;
+  T->exitno = J->exitno;
   setgcrefr(T->nextgc, J2G(J)->gc.root);
   setgcrefp(J2G(J)->gc.root, T);
   newwhite(J2G(J), T);
@@ -262,6 +264,8 @@ int lj_trace_flushall(lua_State *L)
   }
   /* Clear penalty cache. */
   memset(J->penalty, 0, sizeof(J->penalty));
+  /* Reset hotcounts. */
+  lj_dispatch_init_hotcount(J2G(J));
   /* Free the whole machine code and invalidate all exit stub groups. */
   lj_mcode_free(J);
   memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup));
@@ -333,7 +337,7 @@ static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
   J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1);
   setmref(J->penalty[i].pc, pc);
 setpenalty:
-  J->penalty[i].val = (uint16_t)val;
+  J->penalty[i].val = val;
   J->penalty[i].reason = e;
   hotcount_set(J2GG(J), pc+1, val);
 }
@@ -399,6 +403,7 @@ static void trace_stop(jit_State *J)
   TraceNo traceno = J->cur.traceno;
   GCtrace *T = J->curfinal;
   lua_State *L;
+  int i;
 
   switch (op) {
   case BC_FORL:
@@ -450,6 +455,11 @@ static void trace_stop(jit_State *J)
   J->postproc = LJ_POST_NONE;
   trace_save(J, T);
 
+  /* Clear any penalty after successful recording. */
+  for (i = 0; i < PENALTY_SLOTS; i++)
+    if (mref(J->penalty[i].pc, const BCIns) == pc)
+      J->penalty[i].val = PENALTY_MIN;
+
   L = J->L;
 }
 

src/lj_vmprofile.c

@@ -46,7 +46,7 @@ int vmprofile_get_profile_size() {
 void vmprofile_set_profile(void *counters) {
   profile = (VMProfile*)counters;
   profile->magic = 0x1d50f007;
-  profile->major = 2;
+  profile->major = 4;
   profile->minor = 0;
 }
 
@@ -56,28 +56,43 @@ void vmprofile_set_profile(void *counters) {
 static void vmprofile_signal(int sig, siginfo_t *si, void *data)
 {
   if (profile != NULL) {
+    int vmstate, trace;     /* sample matrix indices */
     lua_State *L = gco2th(gcref(state.g->cur_L));
-    int vmstate = state.g->vmstate;
-    int trace = ~vmstate == LJ_VMST_GC ? state.g->gcvmstate : vmstate;
-    /* Not in a trace */
-    if (trace < 0) {
-      profile->vm[~vmstate]++;
-    } else {
-      int bucket = trace > LJ_VMPROFILE_TRACE_MAX ? 0 : trace;
-      VMProfileTraceCount *count = &profile->trace[bucket];
-      GCtrace *T = traceref(L2J(L), (TraceNo)trace);
+    /*
+     * The basic job of this function is to select the right indices
+     * into the profile counter matrix. That requires deciding which
+     * logical state the VM is in and which trace the sample should be
+     * attributed to. Heuristics are needed to pick appropriate values.
+     */
+    if (state.g->vmstate > 0) {    /* Running JIT mcode. */
+      GCtrace *T = traceref(L2J(L), (TraceNo)state.g->vmstate);
       intptr_t ip = (intptr_t)((ucontext_t*)data)->uc_mcontext.gregs[REG_RIP];
       ptrdiff_t mcposition = ip - (intptr_t)T->mcode;
-      if (~vmstate == LJ_VMST_GC) {
-        count->gc++;
-      } else if ((mcposition < 0) || (mcposition >= T->szmcode)) {
-        count->other++;
+      if ((mcposition < 0) || (mcposition >= T->szmcode)) {
+        vmstate = LJ_VMST_FFI;    /* IP is outside the trace mcode. */
       } else if ((T->mcloop != 0) && (mcposition >= T->mcloop)) {
-        count->loop++;
+        vmstate = LJ_VMST_LOOP;   /* IP is inside the mcode loop. */
       } else {
-        count->head++;
+        vmstate = LJ_VMST_HEAD;   /* IP is inside mcode but not loop. */
+      }
+      trace = state.g->vmstate;
+    } else {                    /* Running VM code (not JIT mcode.) */
+      if (~state.g->vmstate == LJ_VMST_GC && state.g->gcvmstate > 0) {
+        /* Special case: GC invoked from JIT mcode. */
+        vmstate = LJ_VMST_JGC;
+        trace = state.g->gcvmstate;
+      } else {
+        /* General case: count towards most recently exited trace. */
+        vmstate = ~state.g->vmstate;
+        trace = state.g->lasttrace;
       }
     }
+    /* Handle overflow from individual trace counters. */
+    trace = trace <= LJ_VMPROFILE_TRACE_MAX ? trace : 0;
+    /* Phew! We have calculated the indices and now we can bump the counter. */
+    lua_assert(vmstate >= 0 && vmstate <= LJ_VMST__MAX);
+    lua_assert(trace >= 0 && trace <= LJ_VMPROFILE_TRACE_MAX);
+    profile->count[trace][vmstate]++;
   }
 }
 
@@ -133,7 +148,7 @@ LUA_API int luaJIT_vmprofile_close(lua_State *L, void *ud)
 LUA_API int luaJIT_vmprofile_select(lua_State *L, void *ud)
 {
   setlightudV(L->base, checklightudptr(L, profile));
-  profile = (VMProfile *)ud;
+  vmprofile_set_profile(ud);
   return 1;
 }