Concurrent sweep

man/mono.1

@@ -1129,9 +1129,16 @@ to 100 percent.  A value of 0 turns evacuation off.
 .TP
 \fB(no-)lazy-sweep\fR
 Enables or disables lazy sweep for the Mark&Sweep collector.  If
-enabled, the sweep phase of the garbage collection is done piecemeal
-whenever the need arises, typically during nursery collections.  Lazy
-sweeping is enabled by default.
+enabled, the sweeping of individual major heap blocks is done
+piecemeal whenever the need arises, typically during nursery
+collections.  Lazy sweeping is enabled by default.
+.TP
+\fB(no-)concurrent-sweep\fR
+Enables or disables concurrent sweep for the Mark&Sweep collector.  If
+enabled, the iteration of all major blocks to determine which ones can
+be freed and which ones have to be kept and swept, is done
+concurrently with the running program.  Concurrent sweeping is enabled
+by default.
 .TP
 \fBstack-mark=\fImark-mode\fR
 Specifies how application threads should be scanned. Options are

mono/metadata/Makefile.am

@@ -291,6 +291,8 @@ sgen_sources = \
 	sgen-layout-stats.h	\
 	sgen-qsort.c	\
 	sgen-qsort.h	\
+	sgen-thread-pool.c	\
+	sgen-thread-pool.h	\
 	sgen-tagged-pointer.h
 
 libmonoruntime_la_SOURCES = $(common_sources) $(gc_dependent_sources) $(null_gc_sources) $(boehm_sources)

mono/metadata/sgen-gc.h

@@ -169,6 +169,7 @@ extern LOCK_DECLARE (sgen_interruption_mutex);
 #define UNLOCK_INTERRUPTION mono_mutex_unlock (&sgen_interruption_mutex)
 
 /* FIXME: Use InterlockedAdd & InterlockedAdd64 to reduce the CAS cost. */
+#define SGEN_CAS	InterlockedCompareExchange
 #define SGEN_CAS_PTR	InterlockedCompareExchangePointer
 #define SGEN_ATOMIC_ADD(x,i)	do {					\
 		int __old_x;						\
@@ -391,8 +392,6 @@ gboolean sgen_resume_thread (SgenThreadInfo *info);
 void sgen_wait_for_suspend_ack (int count);
 void sgen_os_init (void);
 
-gboolean sgen_is_worker_thread (MonoNativeThreadId thread);
-
 void sgen_update_heap_boundaries (mword low, mword high);
 
 void sgen_scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data, gboolean allow_flags);
@@ -420,7 +419,7 @@ enum {
 	INTERNAL_MEM_MS_BLOCK_INFO_SORT,
 	INTERNAL_MEM_EPHEMERON_LINK,
 	INTERNAL_MEM_WORKER_DATA,
-	INTERNAL_MEM_WORKER_JOB_DATA,
+	INTERNAL_MEM_THREAD_POOL_JOB,
 	INTERNAL_MEM_BRIDGE_DATA,
 	INTERNAL_MEM_OLD_BRIDGE_HASH_TABLE,
 	INTERNAL_MEM_OLD_BRIDGE_HASH_TABLE_ENTRY,
@@ -432,7 +431,6 @@ enum {
 	INTERNAL_MEM_TARJAN_BRIDGE_HASH_TABLE_ENTRY,
 	INTERNAL_MEM_TARJAN_OBJ_BUCKET,
 	INTERNAL_MEM_BRIDGE_DEBUG,
-	INTERNAL_MEM_JOB_QUEUE_ENTRY,
 	INTERNAL_MEM_TOGGLEREF_DATA,
 	INTERNAL_MEM_CARDTABLE_MOD_UNION,
 	INTERNAL_MEM_BINARY_PROTOCOL,
@@ -612,12 +610,13 @@ void sgen_split_nursery_init (SgenMinorCollector *collector);
 /* Updating references */
 
 #ifdef SGEN_CHECK_UPDATE_REFERENCE
+gboolean sgen_thread_pool_is_thread_pool_thread (MonoNativeThreadId some_thread) MONO_INTERNAL;
 static inline void
 sgen_update_reference (void **p, void *o, gboolean allow_null)
 {
 	if (!allow_null)
 		SGEN_ASSERT (0, o, "Cannot update a reference with a NULL pointer");
-	SGEN_ASSERT (0, !sgen_is_worker_thread (mono_native_thread_id_get ()), "Can't update a reference in the worker thread");
+	SGEN_ASSERT (0, !sgen_thread_pool_is_thread_pool_thread (mono_native_thread_id_get ()), "Can't update a reference in the worker thread");
 	*p = o;
 }
 
@@ -653,6 +652,7 @@ typedef struct _SgenMajorCollector SgenMajorCollector;
 struct _SgenMajorCollector {
 	size_t section_size;
 	gboolean is_concurrent;
+	gboolean needs_thread_pool;
 	gboolean supports_cardtable;
 	gboolean sweeps_lazily;
 
@@ -673,7 +673,13 @@ struct _SgenMajorCollector {
 
 	void* (*alloc_object) (MonoVTable *vtable, size_t size, gboolean has_references);
 	void (*free_pinned_object) (char *obj, size_t size);
+
+	/*
+	 * This is used for domain unloading, heap walking from the logging profiler, and
+	 * debugging.  Can assume the world is stopped.
+	 */
 	void (*iterate_objects) (IterateObjectsFlags flags, IterateObjectCallbackFunc callback, void *data);
+
 	void (*free_non_pinned_object) (char *obj, size_t size);
 	void (*pin_objects) (SgenGrayQueue *queue);
 	void (*pin_major_object) (char *obj, SgenGrayQueue *queue);
@@ -682,8 +688,9 @@ struct _SgenMajorCollector {
 	void (*update_cardtable_mod_union) (void);
 	void (*init_to_space) (void);
 	void (*sweep) (void);
-	gboolean (*have_finished_sweeping) (void);
-	void (*free_swept_blocks) (void);
+	gboolean (*have_swept) (void);
+	void (*finish_sweeping) (void);
+	void (*free_swept_blocks) (size_t allowance);
 	void (*check_scan_starts) (void);
 	void (*dump_heap) (FILE *heap_dump_file);
 	gint64 (*get_used_size) (void);
@@ -696,13 +703,10 @@ struct _SgenMajorCollector {
 	gboolean (*obj_is_from_pinned_alloc) (char *obj);
 	void (*report_pinned_memory_usage) (void);
 	size_t (*get_num_major_sections) (void);
+	size_t (*get_bytes_survived_last_sweep) (void);
 	gboolean (*handle_gc_param) (const char *opt);
 	void (*print_gc_param_usage) (void);
-	gboolean (*is_worker_thread) (MonoNativeThreadId thread);
 	void (*post_param_init) (SgenMajorCollector *collector);
-	void* (*alloc_worker_data) (void);
-	void (*init_worker_thread) (void *data);
-	void (*reset_worker_data) (void *data);
 	gboolean (*is_valid_object) (char *object);
 	MonoVTable* (*describe_pointer) (char *pointer);
 	guint8* (*get_cardtable_mod_union_for_object) (char *object);
@@ -959,6 +963,7 @@ typedef struct {
 
 int sgen_stop_world (int generation);
 int sgen_restart_world (int generation, GGTimingInfo *timing);
+gboolean sgen_is_world_stopped (void);
 void sgen_init_stw (void);
 
 /* LOS */

mono/metadata/sgen-internal.c

@@ -134,7 +134,7 @@ description_for_type (int type)
 	case INTERNAL_MEM_MS_BLOCK_INFO_SORT: return "marksweep-block-info-sort";
 	case INTERNAL_MEM_EPHEMERON_LINK: return "ephemeron-link";
 	case INTERNAL_MEM_WORKER_DATA: return "worker-data";
-	case INTERNAL_MEM_WORKER_JOB_DATA: return "worker-job-data";
+	case INTERNAL_MEM_THREAD_POOL_JOB: return "thread-pool-job";
 	case INTERNAL_MEM_BRIDGE_DATA: return "bridge-data";
 	case INTERNAL_MEM_OLD_BRIDGE_HASH_TABLE: return "old-bridge-hash-table";
 	case INTERNAL_MEM_OLD_BRIDGE_HASH_TABLE_ENTRY: return "old-bridge-hash-table-entry";
@@ -146,7 +146,6 @@ description_for_type (int type)
 	case INTERNAL_MEM_BRIDGE_ALIVE_HASH_TABLE: return "bridge-alive-hash-table";
 	case INTERNAL_MEM_BRIDGE_ALIVE_HASH_TABLE_ENTRY: return "bridge-alive-hash-table-entry";
 	case INTERNAL_MEM_BRIDGE_DEBUG: return "bridge-debug";
-	case INTERNAL_MEM_JOB_QUEUE_ENTRY: return "job-queue-entry";
 	case INTERNAL_MEM_TOGGLEREF_DATA: return "toggleref-data";
 	case INTERNAL_MEM_CARDTABLE_MOD_UNION: return "cardtable-mod-union";
 	case INTERNAL_MEM_BINARY_PROTOCOL: return "binary-protocol";