Optimize process state changes

erts/emulator/beam/beam_bp.c

@@ -713,7 +713,7 @@ void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type
     BpDataTime *pbdt = NULL;
 
     ASSERT(p);
-    ASSERT(p->status == P_RUNNING);
+    ASSERT(ERTS_PSFLG_RUNNING & erts_smp_atomic32_read_acqb(&p->state));
 
     /* get previous timestamp and breakpoint
      * from the process psd  */

erts/emulator/beam/beam_emu.c

@@ -1875,13 +1875,12 @@ void process_main(void)
 	 msgp = PEEK_MESSAGE(c_p);
 	 if (msgp)
 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
-	 else {
+	 else
 #endif
+	 {
 	     SET_I((BeamInstr *) Arg(0));
 	     Goto(*I);		/* Jump to a wait or wait_timeout instruction */
-#ifdef ERTS_SMP
 	 }
-#endif
      }
      ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS,
 				   {
@@ -2110,11 +2109,11 @@ void process_main(void)
 	 OpCase(wait_f):
 
 	 wait2: {
-	     ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	     c_p->i = (BeamInstr *) Arg(0); /* L1 */
 	     SWAPOUT;
 	     c_p->arity = 0;
-	     c_p->status = P_WAITING;
+	     erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
+	     ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	     c_p->current = NULL;
 	     goto do_schedule;
@@ -3193,10 +3192,6 @@ void process_main(void)
      c_p->arg_reg[0] = r(0);
      SWAPOUT;
      c_p->i = I;
-     erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
-     if (c_p->status != P_SUSPENDED)
-	 erts_add_to_runq(c_p);
-     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
      goto do_schedule1;
  }
 
@@ -5159,9 +5154,6 @@ void process_main(void)
      c_p->arity = 1; /* One living register (the 'true' return value) */
      SWAPOUT;
      c_p->i = I + 1; /* Next instruction */
-     erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS);
-     erts_add_to_runq(c_p);
-     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
      c_p->current = NULL;
      goto do_schedule;
  }
@@ -6255,24 +6247,20 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re
      */
     erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
     ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
-    if (c_p->msg.len > 0) {
-	erts_add_to_runq(c_p);
-    } else {
+    if (!c_p->msg.len) {
 	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
 	c_p->fvalue = NIL;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_garbage_collect_hibernate(c_p);
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 #ifdef ERTS_SMP
 	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
-	if (c_p->msg.len > 0)
-	    erts_add_to_runq(c_p);
-	else
+	if (!c_p->msg.len)
 #endif
-	    c_p->status = P_WAITING;
+	    erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
+	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
     }
     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
     c_p->current = bif_export[BIF_hibernate_3]->code;

erts/emulator/beam/bif.c

@@ -233,15 +233,17 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
 
     BIF_ERROR(BIF_P, BADARG);
 
- res_no_proc:
-    if (BIF_P->flags & F_TRAPEXIT) {
-	ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN;
-	erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL);
-	erts_smp_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks);
-	BIF_RET(am_true);
+res_no_proc: {
+	erts_aint32_t state = erts_smp_atomic32_read_nob(&BIF_P->state);
+	if (state & ERTS_PSFLG_TRAP_EXIT) {
+	    ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN;
+	    erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL);
+	    erts_smp_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks);
+	    BIF_RET(am_true);
+	}
+	else
+	    BIF_ERROR(BIF_P, EXC_NOPROC);
     }
-    else
-	BIF_ERROR(BIF_P, EXC_NOPROC);
 }
 
 #define ERTS_DEMONITOR_FALSE		2
@@ -1103,8 +1105,9 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3)
 
     if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) {
         /*
-         * If hibernate succeeded, TRAP. The process will be suspended
-         * if status is P_WAITING or continue (if any message was in the queue).
+         * If hibernate succeeded, TRAP. The process will be wait in a
+         * hibernated state if its state is inactive (!ERTS_PSFLG_ACTIVE);
+         * otherwise, continue executing (if any message was in the queue).
          */
         BIF_TRAP_CODE_PTR_(BIF_P, BIF_P->i);
     }
@@ -1499,14 +1502,13 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
       BIF_RET(old_value);
    }
    else if (BIF_ARG_1 == am_priority) {
-       erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS);
        old_value = erts_set_process_priority(BIF_P, BIF_ARG_2);
-       erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
        if (old_value == THE_NON_VALUE)
 	   goto error;
        BIF_RET(old_value);
    }
    else if (BIF_ARG_1 == am_trap_exit) {
+       erts_aint32_t state;
        Uint trap_exit;
        if (BIF_ARG_2 == am_true) {
 	   trap_exit = 1;
@@ -1521,59 +1523,52 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
 	*       For more info, see implementation of erts_send_exit_signal().
 	*/
        erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS);
+       ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)
+			  & erts_proc_lc_my_proc_locks(BIF_P));
        ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P,
 				     ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
-       old_value = ERTS_PROC_IS_TRAPPING_EXITS(BIF_P) ? am_true : am_false;
-       if (trap_exit) {
-	   ERTS_PROC_SET_TRAP_EXIT(BIF_P);
-       } else {
-	   ERTS_PROC_UNSET_TRAP_EXIT(BIF_P);
-       }
+       if (trap_exit)
+	   state = erts_smp_atomic32_read_bor_nob(&BIF_P->state,
+						  ERTS_PSFLG_TRAP_EXIT);
+       else
+	   state = erts_smp_atomic32_read_band_nob(&BIF_P->state,
+						   ~ERTS_PSFLG_TRAP_EXIT);
+       old_value = (state & ERTS_PSFLG_TRAP_EXIT) ? am_true : am_false;
        erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
        BIF_RET(old_value);
    }
    else if (BIF_ARG_1 == am_scheduler) {
-       int yield;
-       ErtsRunQueue *old;
-       ErtsRunQueue *new;
+       ErtsRunQueue *old, *new, *curr;
        Sint sched;
+       erts_aint32_t state;
+
        if (!is_small(BIF_ARG_2))
 	   goto error;
        sched = signed_val(BIF_ARG_2);
        if (sched < 0 || erts_no_schedulers < sched)
 	   goto error;
-       erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS);
-       old = BIF_P->bound_runq;
-#ifdef ERTS_SMP
-       ASSERT(!old || old == BIF_P->run_queue);
-#endif
-       new = !sched ? NULL : erts_schedid2runq(sched);
-#ifndef ERTS_SMP
-       yield = 0;
-#else
-       if (new == old)
-	   yield = 0;
+
+       if (sched == 0) {
+	   new = NULL;
+	   state = erts_smp_atomic32_read_band_mb(&BIF_P->state,
+						  ~ERTS_PSFLG_BOUND);
+       }
        else {
-	   ErtsRunQueue *curr = BIF_P->run_queue;
-	   if (!new)
-	       erts_smp_runq_lock(curr);
-	   else
-	       erts_smp_runqs_lock(curr, new);
-	   yield = new && BIF_P->run_queue != new;
-#endif
-	   BIF_P->bound_runq = new;
+	   new = erts_schedid2runq(sched);
 #ifdef ERTS_SMP
-	   if (new)
-	       BIF_P->run_queue = new;
-	   if (!new)
-	       erts_smp_runq_unlock(curr);
-	   else
-	       erts_smp_runqs_unlock(curr, new);
-       }
+	   erts_atomic_set_nob(&BIF_P->run_queue, (erts_aint_t) new);
 #endif
-       erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
+	   state = erts_smp_atomic32_read_bor_mb(&BIF_P->state,
+						 ERTS_PSFLG_BOUND);
+       }
+
+       curr = ERTS_GET_SCHEDULER_DATA_FROM_PROC(BIF_P)->run_queue;
+       old = (ERTS_PSFLG_BOUND & state) ? curr : NULL;
+
+       ASSERT(!old || old == curr);
+
        old_value = old ? make_small(old->ix+1) : make_small(0);
-       if (yield)
+       if (new && new != curr)
 	   ERTS_BIF_YIELD_RETURN_X(BIF_P, old_value, am_scheduler);
        else
 	   BIF_RET(old_value);

erts/emulator/beam/break.c

@@ -73,8 +73,8 @@ process_info(int to, void *to_arg)
     for (i = 0; i < erts_max_processes; i++) {
 	Process *p = erts_pix2proc(i);
 	if (p && p->i != ENULL) {
-	   if (p->status != P_EXITING)
-	       print_process_info(to, to_arg, p);
+	    if (!ERTS_PROC_IS_EXITING(p))
+		print_process_info(to, to_arg, p);
 	}
     }
 
@@ -99,11 +99,20 @@ process_killer(void)
 		if ((j = sys_get_key(0)) <= 0)
 		    erl_exit(0, "");
 		switch(j) {
-		case 'k':
-		    if (rp->status == P_WAITING) {
-			ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
-			erts_smp_proc_inc_refc(rp);
-			erts_smp_proc_lock(rp, rp_locks);
+		case 'k': {
+		    ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
+		    erts_aint32_t state;
+		    erts_smp_proc_inc_refc(rp);
+		    erts_smp_proc_lock(rp, rp_locks);
+		    state = erts_smp_atomic32_read_acqb(&rp->state);
+		    if (state & (ERTS_PSFLG_FREE
+				  | ERTS_PSFLG_EXITING
+				  | ERTS_PSFLG_ACTIVE
+				  | ERTS_PSFLG_IN_RUNQ
+				  | ERTS_PSFLG_RUNNING)) {
+			erts_printf("Can only kill WAITING processes this way\n");
+		    }
+		    else {
 			(void) erts_send_exit_signal(NULL,
 						     NIL,
 						     rp,
@@ -112,12 +121,10 @@ process_killer(void)
 						     NIL,
 						     NULL,
 						     0);
-			erts_smp_proc_unlock(rp, rp_locks);
-			erts_smp_proc_dec_refc(rp);
 		    }
-		    else
-			erts_printf("Can only kill WAITING processes this way\n");
-
+		    erts_smp_proc_unlock(rp, rp_locks);
+		    erts_smp_proc_dec_refc(rp);
+		}
 		case 'n': br = 1; break;
 		case 'r': return;
 		default: return;
@@ -186,38 +193,34 @@ print_process_info(int to, void *to_arg, Process *p)
     int garbing = 0;
     int running = 0;
     struct saved_calls *scb;
+    erts_aint32_t state;
 
     /* display the PID */
     erts_print(to, to_arg, "=proc:%T\n", p->id);
 
     /* Display the state */
     erts_print(to, to_arg, "State: ");
-    switch (p->status) {
-    case P_FREE:
+
+    state = erts_smp_atomic32_read_acqb(&p->state);
+    if (state & ERTS_PSFLG_FREE)
 	erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */
-	break;
-    case P_RUNABLE:
-	erts_print(to, to_arg, "Scheduled\n");
-	break;
-    case P_WAITING:
-	erts_print(to, to_arg, "Waiting\n");
-	break;
-    case P_SUSPENDED:
-	erts_print(to, to_arg, "Suspended\n");
-	break;
-    case P_RUNNING:
-	erts_print(to, to_arg, "Running\n");
-	running = 1;
-	break;
-    case P_EXITING:
+    else if (state & ERTS_PSFLG_EXITING)
 	erts_print(to, to_arg, "Exiting\n");
-	break;
-    case P_GARBING:
-	erts_print(to, to_arg, "Garbing\n");
+    else if (state & ERTS_PSFLG_GC) {
 	garbing = 1;
 	running = 1;
-	break;
+	erts_print(to, to_arg, "Garbing\n");
+    }
+    else if (state & ERTS_PSFLG_SUSPENDED)
+	erts_print(to, to_arg, "Suspended\n");
+    else if (state & ERTS_PSFLG_RUNNING) {
+	running = 1;
+	erts_print(to, to_arg, "Running\n");
     }
+    else if (state & ERTS_PSFLG_ACTIVE)
+	erts_print(to, to_arg, "Scheduled\n");
+    else
+	erts_print(to, to_arg, "Waiting\n");
 
     /*
      * If the process is registered as a global process, display the

erts/emulator/beam/erl_alloc.types

@@ -336,6 +336,8 @@ type	SL_PTIMER	SHORT_LIVED	SYSTEM		ptimer_sl
 type	LL_PTIMER	STANDARD	SYSTEM		ptimer_ll
 type	SYS_MSG_Q	SHORT_LIVED	PROCESSES	system_messages_queue
 type	FP_EXCEPTION	LONG_LIVED	SYSTEM		fp_exception
+type	LL_MPATHS	LONG_LIVED	SYSTEM		ll_migration_paths
+type	SL_MPATHS	SHORT_LIVED	SYSTEM		sl_migration_paths
 +endif
 
 +if hipe

erts/emulator/beam/erl_alloc_util.c

@@ -832,14 +832,14 @@ init_dd_queue(ErtsAllctrDDQueue_t *ddq)
 static ERTS_INLINE erts_aint_t
 ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr)
 {
-    erts_aint_t ilast, itmp;
+    erts_aint_t first_ilast, ilast, itmp;
     ErtsAllctrDDBlock_t *this = ptr;
 
     erts_atomic_init_nob(&this->atmc_next, ERTS_AINT_NULL);
 
     /* Enqueue at end of list... */
 
-    ilast = erts_atomic_read_nob(&ddq->tail.data.last);
+    first_ilast = ilast = erts_atomic_read_nob(&ddq->tail.data.last);
     while (1) {
 	ErtsAllctrDDBlock_t *last = (ErtsAllctrDDBlock_t *) ilast;
 	itmp = erts_atomic_cmpxchg_mb(&last->atmc_next,
@@ -853,8 +853,11 @@ ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr)
     /* Move last pointer forward... */
     while (1) {
 	if (erts_atomic_read_rb(&this->atmc_next) != ERTS_AINT_NULL) {
-	    /* Someone else will move it forward */
-	    return erts_atomic_read_rb(&ddq->tail.data.last);
+	    ilast = erts_atomic_read_rb(&ddq->tail.data.last);
+	    if (first_ilast != ilast) {
+		/* Someone else will move it forward */
+		return ilast;
+	    }
 	}
 	itmp = erts_atomic_cmpxchg_mb(&ddq->tail.data.last,
 				      (erts_aint_t) this,