[runtime] adding additional assert to highlight source of a race

We've seen that error for quite some time:

> `* Assertion at ../../mono/utils/mono-os-mutex.h:71, condition `res != EINVAL' not met`

It looks like there's a race between `finalizer_thread()` and `mono_gc_cleanup()` around
`reference_queue_mutex`. The source of the problem is, `finalizer_thread` could end up
being alive, although `mono_gc_cleanup()` made some effort to kill it and then destroys
`reference_queue_mutex`, while the finalizer thread is still using it in
`reference_queue_proccess_all()`.

> $ MONO_GC_DEBUG=bridge=Bridge MONO_GC_PARAMS=minor=split MONO_ENV_OPTIONS=--gc=sgen MONO_PATH=/home/lewurm/monoperf/mono/mcs/class/lib/net_4_x ../../mono/mini/mono sgen-bridge-major-fragmentation.exe --optimize=all --debug
> [...]
> [12/08/2015 01:49:31] done
> Shutting down finalizer thread timed out.
> * Assertion at gc.c:867, condition `finalizer_thread_exited' not met
>
> Stacktrace:
>
>
> Native stacktrace:
>
>
> Debug info from gdb:
>
> [New LWP 7623]
> [New LWP 7622]
> [Thread debugging using libthread_db enabled]
> Using host libthread_db library "/lib/arm-linux-gnueabihf/libthread_db.so.1".
> 0x4043b5f4 in __libc_do_syscall () from /lib/arm-linux-gnueabihf/libpthread.so.0
>   Id   Target Id         Frame
>   3    Thread 0x417ff430 (LWP 7622) "mono" 0x4043b5f4 in __libc_do_syscall () from /lib/arm-linux-gnueabihf/libpthread.so.0
>   2    Thread 0x42573430 (LWP 7623) "Finalizer" 0x0021482a in finalizers_with_predicate (predicate=0x1fd121 <object_in_domain_predicate>, user_data=0x37a348, out_array=0x42572c70, out_size=64, hash_table=0x338f68 <major_finalizable_hash>) at sgen-fin-weak-hash.c:581
> * 1    Thread 0x406c4250 (LWP 7620) "mono" 0x4043b5f4 in __libc_do_syscall () from /lib/arm-linux-gnueabihf/libpthread.so.0
>
> Thread 3 (Thread 0x417ff430 (LWP 7622)):
> #0  0x4043b5f4 in __libc_do_syscall () from /lib/arm-linux-gnueabihf/libpthread.so.0
> #1  0x404371d8 in pthread_cond_wait@@GLIBC_2.4 () from /lib/arm-linux-gnueabihf/libpthread.so.0
> #2  0x00242266 in mono_os_cond_wait (cond=0x346e98 <work_cond>, mutex=0x346e80 <lock>) at ../../mono/utils/mono-os-mutex.h:105
> mono#3  0x00242c1a in thread_func (thread_data=0x0) at sgen-thread-pool.c:118
> mono#4  0x40433fbc in start_thread () from /lib/arm-linux-gnueabihf/libpthread.so.0
> mono#5  0x405a0b3c in ?? () from /lib/arm-linux-gnueabihf/libc.so.6
> Backtrace stopped: previous frame identical to this frame (corrupt stack?)
>
> Thread 2 (Thread 0x42573430 (LWP 7623)):
> #0  0x0021482a in finalizers_with_predicate (predicate=0x1fd121 <object_in_domain_predicate>, user_data=0x37a348, out_array=0x42572c70, out_size=64, hash_table=0x338f68 <major_finalizable_hash>) at sgen-fin-weak-hash.c:581
> #1  0x00214894 in sgen_gather_finalizers_if (predicate=0x1fd121 <object_in_domain_predicate>, user_data=0x37a348, out_array=0x42572c70, out_size=64) at sgen-fin-weak-hash.c:622
> #2  0x001fd16a in mono_gc_finalizers_for_domain (domain=0x37a348, out_array=0x42572c70, out_size=64) at sgen-mono.c:546
> mono#3  0x001ca2a4 in finalize_domain_objects (req=0x47ba68) at gc.c:678
> mono#4  0x001ca3b6 in finalizer_thread (unused=0x0) at gc.c:730
> mono#5  0x001a1392 in start_wrapper_internal (data=0x3b1100) at threads.c:713
> mono#6  0x001a1430 in start_wrapper (data=0x3b1100) at threads.c:760
> mono#7  0x0027097a in inner_start_thread (arg=0xbed58de0) at mono-threads-posix.c:92
> mono#8  0x40433fbc in start_thread () from /lib/arm-linux-gnueabihf/libpthread.so.0
> mono#9  0x405a0b3c in ?? () from /lib/arm-linux-gnueabihf/libc.so.6
> Backtrace stopped: previous frame identical to this frame (corrupt stack?)
>
> Thread 1 (Thread 0x406c4250 (LWP 7620)):
> #0  0x4043b5f4 in __libc_do_syscall () from /lib/arm-linux-gnueabihf/libpthread.so.0
> #1  0x4043a396 in waitpid () from /lib/arm-linux-gnueabihf/libpthread.so.0
> #2  0x000d1b24 in mono_handle_native_sigsegv (signal=6, ctx=0xbed58a80, info=0xbed58a00) at mini-exceptions.c:2235
> mono#3  0x00112424 in sigabrt_signal_handler (_dummy=6, _info=0xbed58a00, context=0xbed58a80) at mini-posix.c:218
> mono#4  <signal handler called>
> mono#5  0x405248e6 in ?? () from /lib/arm-linux-gnueabihf/libc.so.6
> mono#6  0x405330fe in raise () from /lib/arm-linux-gnueabihf/libc.so.6
> mono#7  0x40535956 in abort () from /lib/arm-linux-gnueabihf/libc.so.6
> mono#8  0x00277eaa in monoeg_log_default_handler (log_domain=0x0, log_level=G_LOG_LEVEL_ERROR, message=0x47bb28 "* Assertion at gc.c:867, condition `finalizer_thread_exited' not met\n", unused_data=0x0) at goutput.c:233
> mono#9  0x00277dca in monoeg_g_logv (log_domain=0x0, log_level=G_LOG_LEVEL_ERROR, format=0x3089b8 "* Assertion at %s:%d, condition `%s' not met\n", args=...) at goutput.c:113
> mono#10 0x00277e32 in monoeg_assertion_message (format=0x3089b8 "* Assertion at %s:%d, condition `%s' not met\n") at goutput.c:133
> mono#11 0x001ca786 in mono_gc_cleanup () at gc.c:867
> mono#12 0x001c1d24 in mono_runtime_cleanup (domain=0x37a348) at appdomain.c:356
> mono#13 0x0001dec0 in mini_cleanup (domain=0x37a348) at mini-runtime.c:3560
> mono#14 0x000a672c in mono_main (argc=5, argv=0x358098) at driver.c:2065
> mono#15 0x00017c4e in mono_main_with_options (argc=5, argv=0x358098) at main.c:20
> mono#16 0x00017c7e in main (argc=4, argv=0xbed591b4) at main.c:53

=================================================================
Got a SIGABRT while executing native code. This usually indicates
a fatal error in the mono runtime or one of the native libraries
used by your application.
=================================================================

mono/metadata/gc.c

@@ -864,13 +864,15 @@ mono_gc_cleanup (void)
 
 				mono_thread_join (GUINT_TO_POINTER (gc_thread->tid));
 			}
+			g_assert (finalizer_thread_exited);
 		}
 		gc_thread = NULL;
 		mono_gc_base_cleanup ();
 	}
 
 	mono_reference_queue_cleanup ();
 
+	// fprintf (stderr, "destory finalizer and reference queue mutex\n");
 	mono_coop_mutex_destroy (&finalizer_mutex);
 	mono_coop_mutex_destroy (&reference_queue_mutex);
 }

mono/tests/test-driver

@@ -47,6 +47,9 @@ my $cpid = fork ();
 if (!defined ($cpid)) {
 	$res = system("$interpreter @ARGV $test_binary 2>$stderr 1>$stdout");
 } elsif ($cpid == 0) {
+	    print "that's it >>>>>>>>\n";
+	    print "$interpreter @ARGV $test_binary 2>$stderr 1>$stdout\n";
+	    print "\n<<<<<<<<<\n";
 	exec ("$interpreter @ARGV $test_binary 2>$stderr 1>$stdout") || die "Cannot exec: $!";
 } else {
 	# in the parent, setup the alarm