Merge #140 branch 'vmprofile-v4' into integrate

src/lj_obj.h

@@ -438,13 +438,22 @@ typedef struct GCtab {
 
 /* VM states. */
 enum {
+  /* VM states. */
   LJ_VMST_INTERP,	/* Interpreter. */
   LJ_VMST_C,		/* C function. */
   LJ_VMST_GC,		/* Garbage collector. */
   LJ_VMST_EXIT,		/* Trace exit handler. */
   LJ_VMST_RECORD,	/* Trace recorder. */
   LJ_VMST_OPT,		/* Optimizer. */
   LJ_VMST_ASM,		/* Assembler. */
+  /* JIT trace states.
+  ** These are "abstract" states that logically exist but are never
+  ** directly used for the value of global_State.vmstate.
+  */
+  LJ_VMST_HEAD,		/* Trace mcode before loop */
+  LJ_VMST_LOOP,		/* Trace mcode inside loop */
+  LJ_VMST_JGC,		/* GC invoked from JIT mcode. */
+  LJ_VMST_FFI,		/* Other code outside trace mcode */
   LJ_VMST__MAX
 };
 
@@ -518,6 +527,7 @@ typedef struct global_State {
   GCState gc;		/* Garbage collector. */
   volatile int32_t vmstate;   /* VM state or current JIT code trace number. */
   volatile int32_t gcvmstate; /* Previous VM state (only when state is GC). */
+  volatile int32_t lasttrace; /* VM state before exit to interpreter. */
   SBuf tmpbuf;		/* Temporary string buffer. */
   GCstr strempty;	/* Empty string. */
   uint8_t stremptyz;	/* Zero terminator of empty string. */

src/lj_vmprofile.c

@@ -46,7 +46,7 @@ int vmprofile_get_profile_size() {
 void vmprofile_set_profile(void *counters) {
   profile = (VMProfile*)counters;
   profile->magic = 0x1d50f007;
-  profile->major = 2;
+  profile->major = 4;
   profile->minor = 0;
 }
 
@@ -56,28 +56,43 @@ void vmprofile_set_profile(void *counters) {
 static void vmprofile_signal(int sig, siginfo_t *si, void *data)
 {
   if (profile != NULL) {
+    int vmstate, trace;     /* sample matrix indices */
     lua_State *L = gco2th(gcref(state.g->cur_L));
-    int vmstate = state.g->vmstate;
-    int trace = ~vmstate == LJ_VMST_GC ? state.g->gcvmstate : vmstate;
-    /* Not in a trace */
-    if (trace < 0) {
-      profile->vm[~vmstate]++;
-    } else {
-      int bucket = trace > LJ_VMPROFILE_TRACE_MAX ? 0 : trace;
-      VMProfileTraceCount *count = &profile->trace[bucket];
-      GCtrace *T = traceref(L2J(L), (TraceNo)trace);
+    /*
+     * The basic job of this function is to select the right indices
+     * into the profile counter matrix. That requires deciding which
+     * logical state the VM is in and which trace the sample should be
+     * attributed to. Heuristics are needed to pick appropriate values.
+     */
+    if (state.g->vmstate > 0) {    /* Running JIT mcode. */
+      GCtrace *T = traceref(L2J(L), (TraceNo)state.g->vmstate);
       intptr_t ip = (intptr_t)((ucontext_t*)data)->uc_mcontext.gregs[REG_RIP];
       ptrdiff_t mcposition = ip - (intptr_t)T->mcode;
-      if (~vmstate == LJ_VMST_GC) {
-        count->gc++;
-      } else if ((mcposition < 0) || (mcposition >= T->szmcode)) {
-        count->other++;
+      if ((mcposition < 0) || (mcposition >= T->szmcode)) {
+        vmstate = LJ_VMST_FFI;    /* IP is outside the trace mcode. */
       } else if ((T->mcloop != 0) && (mcposition >= T->mcloop)) {
-        count->loop++;
+        vmstate = LJ_VMST_LOOP;   /* IP is inside the mcode loop. */
       } else {
-        count->head++;
+        vmstate = LJ_VMST_HEAD;   /* IP is inside mcode but not loop. */
+      }
+      trace = state.g->vmstate;
+    } else {                    /* Running VM code (not JIT mcode.) */
+      if (~state.g->vmstate == LJ_VMST_GC && state.g->gcvmstate > 0) {
+        /* Special case: GC invoked from JIT mcode. */
+        vmstate = LJ_VMST_JGC;
+        trace = state.g->gcvmstate;
+      } else {
+        /* General case: count towards most recently exited trace. */
+        vmstate = ~state.g->vmstate;
+        trace = state.g->lasttrace;
       }
     }
+    /* Handle overflow from individual trace counters. */
+    trace = trace <= LJ_VMPROFILE_TRACE_MAX ? trace : 0;
+    /* Phew! We have calculated the indices and now we can bump the counter. */
+    lua_assert(vmstate >= 0 && vmstate <= LJ_VMST__MAX);
+    lua_assert(trace >= 0 && trace <= LJ_VMPROFILE_TRACE_MAX);
+    profile->count[trace][vmstate]++;
   }
 }
 
@@ -133,7 +148,7 @@ LUA_API int luaJIT_vmprofile_close(lua_State *L, void *ud)
 LUA_API int luaJIT_vmprofile_select(lua_State *L, void *ud)
 {
   setlightudV(L->base, checklightudptr(L, profile));
-  profile = (VMProfile *)ud;
+  vmprofile_set_profile(ud);
   return 1;
 }
 

src/lj_vmprofile.h

@@ -6,28 +6,26 @@
 #ifndef _LJ_VMPROFILE_H
 #define _LJ_VMPROFILE_H
 
-
 /* Counters are 64-bit to avoid overflow even in long running processes. */
 typedef uint64_t VMProfileCount;
 
 /* Maximum trace number for distinct counter buckets. Traces with
-   higher numbers will be counted together in bucket zero. */
+   higher numbers will be counted together in a shared overflow bucket. */
 #define LJ_VMPROFILE_TRACE_MAX 4096
 
-/* Traces have separate counters for different machine code regions. */
-typedef struct VMProfileTraceCount {
-  VMProfileCount head;          /* Head of the trace (non-looping part) */
-  VMProfileCount loop;          /* Loop of the trace */
-  VMProfileCount other;         /* Outside the trace mcode (unidentified) */
-  VMProfileCount gc;            /* Garbage collection from this trace. */
-} VMProfileTraceCount;
-
 /* Complete set of counters for VM and traces. */
 typedef struct VMProfile {
   uint32_t magic;               /* 0x1d50f007 */
-  uint16_t major, minor;        /* 2, 0 */
-  VMProfileCount vm[LJ_VMST__MAX];
-  VMProfileTraceCount trace[LJ_VMPROFILE_TRACE_MAX+1];
+  uint16_t major, minor;        /* 4, 0 */
+  /* Profile counters are stored in a 2D matrix of count[trace][state].
+  **
+  ** The profiler attempts to attribute each sample to one vmstate and
+  ** one trace. The vmstate is an LJ_VMST_* constant. The trace is
+  ** either 1..4096 (counter for one individual trace) or 0 (shared
+  ** counter for all higher-numbered traces and for samples that can't
+  ** be attributed to a specific trace at all.)
+  **/
+  VMProfileCount count[LJ_VMPROFILE_TRACE_MAX+1][LJ_VMST__MAX];
 } VMProfile;
 
 /* Functions that should be accessed via FFI. */