diff --git a/mono/metadata/gc-internal.h b/mono/metadata/gc-internal.h
index f21e99b0dfb54..3a774beb44916 100644
--- a/mono/metadata/gc-internal.h
+++ b/mono/metadata/gc-internal.h
@@ -339,6 +339,7 @@ struct _RefQueueEntry {
 #else
 	guint32 gchandle;
 #endif
+	MonoDomain *domain;
 	void *user_data;
 	RefQueueEntry *next;
 };
diff --git a/mono/metadata/gc.c b/mono/metadata/gc.c
index 3337b02d5d749..e8c2cb724d843 100644
--- a/mono/metadata/gc.c
+++ b/mono/metadata/gc.c
@@ -1396,13 +1396,10 @@ reference_queue_clear_for_domain (MonoDomain *domain)
 		RefQueueEntry *entry;
 		while ((entry = *iter)) {
 			MonoObject *obj;
+			if (entry->domain == domain) {
 #ifdef HAVE_SGEN_GC
-			obj = mono_gc_weak_link_get (&entry->dis_link);
-			if (obj && mono_object_domain (obj) == domain) {
 				mono_gc_weak_link_remove (&entry->dis_link, TRUE);
 #else
-			obj = mono_gchandle_get_target (entry->gchandle);
-			if (obj && mono_object_domain (obj) == domain) {
 				mono_gchandle_free ((guint32)entry->gchandle);
 #endif
 				ref_list_remove_element (iter, entry);
@@ -1458,6 +1455,7 @@ mono_gc_reference_queue_add (MonoReferenceQueue *queue, MonoObject *obj, void *u
 
 	entry = g_new0 (RefQueueEntry, 1);
 	entry->user_data = user_data;
+	entry->domain = mono_object_domain (obj);
 
 #ifdef HAVE_SGEN_GC
 	mono_gc_weak_link_add (&entry->dis_link, obj, TRUE);