Merge branch 'lukas/erts/tracing/bif_return_to_trace_fix/OTP-13734' i…

…nto maint

* lukas/erts/tracing/bif_return_to_trace_fix/OTP-13734:
  erts: After a call to a tracer nif, reset htop
  erts: Check if return_to trace is enabled for bif

erts/emulator/beam/beam_bp.c

@@ -858,7 +858,8 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 	if (flags & MATCH_SET_RX_TRACE) {
 	    erts_trace_return(p, ep->code, result, &ERTS_TRACER(p));
 	}
-	if (flags & MATCH_SET_RETURN_TO_TRACE) {
+	if (flags & MATCH_SET_RETURN_TO_TRACE &&
+            IS_TRACED_FL(p, F_TRACE_RETURN_TO)) {
 	    /* can only happen if(local)*/
 	    if (applying) {
 		/* Apply of BIF, cp is in calling function */

erts/emulator/beam/erl_nif.c

@@ -3502,6 +3502,7 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
         struct enif_environment_t env;
         ErlHeapFragment *orig_hf = MBUF(p);
         ErlOffHeap orig_oh = MSO(p);
+        Eterm *orig_htop = HEAP_TOP(p);
         ASSERT(is_internal_pid(p->common.id));
         MBUF(p) = NULL;
         clear_offheap(&MSO(p));
@@ -3523,6 +3524,7 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
         /* restore original heap fragment list */
         MBUF(p) = orig_hf;
         MSO(p) = orig_oh;
+        HEAP_TOP(p) = orig_htop;
     } else {
         /* Nif call was done without a process context,
            so we create a phony one. */

erts/emulator/test/trace_local_SUITE.erl

@@ -439,6 +439,14 @@ return_test() ->
     ?RT(?MODULE,slave,2),
     shutdown(),
     ?NM,
+
+    %% Test a regression where turning off return_to tracing
+    %% on yourself would cause a segfault.
+    Pid = setup([call,return_to]),
+    erlang:trace_pattern({'_','_','_'},[],[local]),
+    apply_slave(erlang,trace,[Pid, false, [all]]),
+    shutdown(),
+
     ok.
 
 on_and_off_test() ->