Convert thread list lock from mutex to rwlock.

lib/libthr/thread/thr_affinity.c

@@ -50,8 +50,7 @@ _pthread_setaffinity_np(pthread_t td, size_t cpusetsize, const cpuset_t *cpusetp
 			-1, cpusetsize, cpusetp);
 		if (error == -1)
 			error = errno;
-	} else {
-		THR_THREAD_LOCK(curthread, td);
+	} else if ((error = _thr_find_thread(curthread, td, 0)) == 0) {
 		if (td->state == PS_DEAD) {
 			THR_THREAD_UNLOCK(curthread, td);
 			return (EINVAL);
@@ -73,10 +72,18 @@ _pthread_getaffinity_np(pthread_t td, size_t cpusetsize, cpuset_t *cpusetp)
 	lwpid_t tid;
 	int error;
 
-	tid = TID(td);
-	error = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID,
-		(td == curthread) ? -1 : tid, cpusetsize, cpusetp);
-	if (error == -1)
-		error = errno;
+	if (td == curthread) {
+		error = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID,
+			(td == curthread) ? -1 : tid, cpusetsize, cpusetp);
+		if (error == -1)
+			error = errno;
+	} else if ((error = _thr_find_thread(curthread, td, 0)) == 0) {
+		tid = TID(td);
+		error = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID,
+			(td == curthread) ? -1 : tid, cpusetsize, cpusetp);
+		if (error == -1)
+			error = errno;
+		THR_THREAD_UNLOCK(curthread, td);
+	}
 	return (error);
 }

lib/libthr/thread/thr_attr.c

@@ -132,22 +132,23 @@ _pthread_attr_destroy(pthread_attr_t *attr)
 __weak_reference(_pthread_attr_get_np, pthread_attr_get_np);
 
 int
-_pthread_attr_get_np(pthread_t pid, pthread_attr_t *dst)
+_pthread_attr_get_np(pthread_t pthread, pthread_attr_t *dst)
 {
 	struct pthread *curthread;
 	struct pthread_attr attr;
 	int	ret;
 
-	if (pid == NULL || dst == NULL || *dst == NULL)
+	if (pthread == NULL || dst == NULL || *dst == NULL)
 		return (EINVAL);
 
 	curthread = _get_curthread();
-	if ((ret = _thr_ref_add(curthread, pid, /*include dead*/0)) != 0)
+	if ((ret = _thr_find_thread(curthread, pthread, /*include dead*/0)) != 0)
 		return (ret);
-	attr = pid->attr;
-	if (pid->tlflags & TLFLAGS_DETACHED)
+	attr = pthread->attr;
+	if (pthread->flags & THR_FLAGS_DETACHED)
 		attr.flags |= PTHREAD_DETACHED;
-	_thr_ref_delete(curthread, pid);
+	THR_THREAD_UNLOCK(curthread, pthread);
+
 	memcpy(*dst, &attr, sizeof(struct pthread_attr));
 	/* XXX */
 	(*dst)->cpuset = NULL;

lib/libthr/thread/thr_cancel.c

@@ -60,18 +60,16 @@ _pthread_cancel(pthread_t pthread)
 
 	/*
 	 * POSIX says _pthread_cancel should be async cancellation safe.
-	 * _thr_ref_add and _thr_ref_delete will enter and leave critical
+	 * _thr_find_thread and THR_THREAD_UNLOCK will enter and leave critical
 	 * region automatically.
 	 */
-	if ((ret = _thr_ref_add(curthread, pthread, 0)) == 0) {
-		THR_THREAD_LOCK(curthread, pthread);
+	if ((ret = _thr_find_thread(curthread, pthread, 0)) == 0) {
 		if (!pthread->cancel_pending) {
 			pthread->cancel_pending = 1;
 			if (pthread->state != PS_DEAD)
 				_thr_send_sig(pthread, SIGCANCEL);
 		}
 		THR_THREAD_UNLOCK(curthread, pthread);
-		_thr_ref_delete(curthread, pthread);
 	}
 	return (ret);
 }

lib/libthr/thread/thr_create.c

@@ -125,7 +125,7 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 	new_thread->state = PS_RUNNING;
 
 	if (new_thread->attr.flags & PTHREAD_CREATE_DETACHED)
-		new_thread->tlflags |= TLFLAGS_DETACHED;
+		new_thread->flags |= THR_FLAGS_DETACHED;
 
 	/* Add the new thread. */
 	new_thread->refcount = 1;
@@ -185,39 +185,32 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 			THR_THREAD_LOCK(curthread, new_thread);
 		new_thread->state = PS_DEAD;
 		new_thread->tid = TID_TERMINATED;
+		new_thread->flags |= THR_FLAGS_DETACHED;
+		new_thread->refcount--;
 		if (new_thread->flags & THR_FLAGS_NEED_SUSPEND) {
 			new_thread->cycle++;
 			_thr_umtx_wake(&new_thread->cycle, INT_MAX, 0);
 		}
-		THR_THREAD_UNLOCK(curthread, new_thread);
-		THREAD_LIST_LOCK(curthread);
-		_thread_active_threads--;
-		new_thread->tlflags |= TLFLAGS_DETACHED;
-		_thr_ref_delete_unlocked(curthread, new_thread);
-		THREAD_LIST_UNLOCK(curthread);
+		_thr_try_gc(curthread, new_thread); /* thread lock released */
+		atomic_add_int(&_thread_active_threads, -1);
 	} else if (locked) {
 		if (cpusetp != NULL) {
 			if (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID,
 				TID(new_thread), cpusetsize, cpusetp)) {
 				ret = errno;
 				/* kill the new thread */
 				new_thread->force_exit = 1;
-				THR_THREAD_UNLOCK(curthread, new_thread);
+				new_thread->flags |= THR_FLAGS_DETACHED;
+				_thr_try_gc(curthread, new_thread);
+				 /* thread lock released */
 				goto out;
 			}
 		}
 
 		_thr_report_creation(curthread, new_thread);
 		THR_THREAD_UNLOCK(curthread, new_thread);
-out:
-		if (ret) {
-			THREAD_LIST_LOCK(curthread);
-			new_thread->tlflags |= TLFLAGS_DETACHED;
-			THR_GCLIST_ADD(new_thread);
-			THREAD_LIST_UNLOCK(curthread);
-		}
 	}
-
+out:
 	if (ret)
 		(*thread) = 0;
 	return (ret);

lib/libthr/thread/thr_detach.c

@@ -47,25 +47,21 @@ _pthread_detach(pthread_t pthread)
 	if (pthread == NULL)
 		return (EINVAL);
 
-	THREAD_LIST_LOCK(curthread);
 	if ((rval = _thr_find_thread(curthread, pthread,
 			/*include dead*/1)) != 0) {
-		THREAD_LIST_UNLOCK(curthread);
 		return (rval);
 	}
 
 	/* Check if the thread is already detached or has a joiner. */
-	if ((pthread->tlflags & TLFLAGS_DETACHED) != 0 ||
+	if ((pthread->flags & THR_FLAGS_DETACHED) != 0 ||
 	    (pthread->joiner != NULL)) {
-		THREAD_LIST_UNLOCK(curthread);
+		THR_THREAD_UNLOCK(curthread, pthread);
 		return (EINVAL);
 	}
 
 	/* Flag the thread as detached. */
-	pthread->tlflags |= TLFLAGS_DETACHED;
-	if (pthread->state == PS_DEAD)
-		THR_GCLIST_ADD(pthread);
-	THREAD_LIST_UNLOCK(curthread);
+	pthread->flags |= THR_FLAGS_DETACHED;
+	_thr_try_gc(curthread, pthread); /* thread lock released */
 
 	return (0);
 }

lib/libthr/thread/thr_exit.c

@@ -108,37 +108,34 @@ _pthread_exit_mask(void *status, sigset_t *mask)
 	if (!_thr_isthreaded())
 		exit(0);
 
-	THREAD_LIST_LOCK(curthread);
-	_thread_active_threads--;
-	if (_thread_active_threads == 0) {
-		THREAD_LIST_UNLOCK(curthread);
+	if (atomic_fetchadd_int(&_thread_active_threads, -1) == 1) {
 		exit(0);
 		/* Never reach! */
 	}
-	THREAD_LIST_UNLOCK(curthread);
 
 	/* Tell malloc that the thread is exiting. */
 	_malloc_thread_cleanup();
 
-	THREAD_LIST_LOCK(curthread);
 	THR_LOCK(curthread);
 	curthread->state = PS_DEAD;
 	if (curthread->flags & THR_FLAGS_NEED_SUSPEND) {
 		curthread->cycle++;
 		_thr_umtx_wake(&curthread->cycle, INT_MAX, 0);
 	}
-	THR_UNLOCK(curthread);
 	/*
 	 * Thread was created with initial refcount 1, we drop the
 	 * reference count to allow it to be garbage collected.
 	 */
 	curthread->refcount--;
-	if (curthread->tlflags & TLFLAGS_DETACHED)
-		THR_GCLIST_ADD(curthread);
-	THREAD_LIST_UNLOCK(curthread);
+	_thr_try_gc(curthread, curthread); /* thread lock released */
+
 	if (!curthread->force_exit && SHOULD_REPORT_EVENT(curthread, TD_DEATH))
 		_thr_report_death(curthread);
 
+#if defined(_PTHREADS_INVARIANTS)
+	if (THR_IN_CRITICAL(curthread))
+		PANIC("thread exits with resources held!");
+#endif
 	/*
 	 * Kernel will do wakeup at the address, so joiner thread
 	 * will be resumed if it is sleeping at the address.

lib/libthr/thread/thr_fork.c

@@ -178,13 +178,13 @@ _fork(void)
 		/* Child process */
 		errsave = errno;
 		curthread->cancel_pending = 0;
-		curthread->flags &= ~THR_FLAGS_NEED_SUSPEND;
+		curthread->flags &= ~(THR_FLAGS_NEED_SUSPEND|THR_FLAGS_DETACHED);
 
 		/*
 		 * Thread list will be reinitialized, and later we call
 		 * _libpthread_init(), it will add us back to list.
 		 */
-		curthread->tlflags &= ~(TLFLAGS_IN_TDLIST | TLFLAGS_DETACHED);
+		curthread->tlflags &= ~TLFLAGS_IN_TDLIST;
 
 		/* child is a new kernel thread. */
 		thr_self(&curthread->tid);

lib/libthr/thread/thr_init.c

@@ -111,7 +111,7 @@ struct umutex	_mutex_static_lock = DEFAULT_UMUTEX;
 struct umutex	_cond_static_lock = DEFAULT_UMUTEX;
 struct umutex	_rwlock_static_lock = DEFAULT_UMUTEX;
 struct umutex	_keytable_lock = DEFAULT_UMUTEX;
-struct umutex	_thr_list_lock = DEFAULT_UMUTEX;
+struct urwlock	_thr_list_lock = DEFAULT_URWLOCK;
 struct umutex	_thr_event_lock = DEFAULT_UMUTEX;
 
 int	__pthread_cond_wait(pthread_cond_t *, pthread_mutex_t *);

lib/libthr/thread/thr_join.c

@@ -43,12 +43,12 @@ __weak_reference(_pthread_timedjoin_np, pthread_timedjoin_np);
 
 static void backout_join(void *arg)
 {
-	struct pthread *curthread = _get_curthread();
 	struct pthread *pthread = (struct pthread *)arg;
+	struct pthread *curthread = _get_curthread();
 
-	THREAD_LIST_LOCK(curthread);
+	THR_THREAD_LOCK(curthread, pthread);
 	pthread->joiner = NULL;
-	THREAD_LIST_UNLOCK(curthread);
+	THR_THREAD_LOCK(curthread, pthread);
 }
 
 int
@@ -88,23 +88,23 @@ join_common(pthread_t pthread, void **thread_return,
 	if (pthread == curthread)
 		return (EDEADLK);
 
-	THREAD_LIST_LOCK(curthread);
-	if ((ret = _thr_find_thread(curthread, pthread, 1)) != 0) {
-		ret = ESRCH;
-	} else if ((pthread->tlflags & TLFLAGS_DETACHED) != 0) {
+	if ((ret = _thr_find_thread(curthread, pthread, 1)) != 0)
+		return (ESRCH);
+
+	if ((pthread->flags & THR_FLAGS_DETACHED) != 0) {
 		ret = EINVAL;
 	} else if (pthread->joiner != NULL) {
 		/* Multiple joiners are not supported. */
 		ret = ENOTSUP;
 	}
 	if (ret) {
-		THREAD_LIST_UNLOCK(curthread);
+		THR_THREAD_UNLOCK(curthread, pthread);
 		return (ret);
 	}
 	/* Set the running thread to be the joiner: */
 	pthread->joiner = curthread;
 
-	THREAD_LIST_UNLOCK(curthread);
+	THR_THREAD_UNLOCK(curthread, pthread);
 
 	THR_CLEANUP_PUSH(curthread, backout_join, pthread);
 	_thr_cancel_enter(curthread);
@@ -131,17 +131,16 @@ join_common(pthread_t pthread, void **thread_return,
 	THR_CLEANUP_POP(curthread, 0);
 
 	if (ret == ETIMEDOUT) {
-		THREAD_LIST_LOCK(curthread);
+		THR_THREAD_LOCK(curthread, pthread);
 		pthread->joiner = NULL;
-		THREAD_LIST_UNLOCK(curthread);
+		THR_THREAD_UNLOCK(curthread, pthread);
 	} else {
 		ret = 0;
 		tmp = pthread->ret;
-		THREAD_LIST_LOCK(curthread);
-		pthread->tlflags |= TLFLAGS_DETACHED;
+		THR_THREAD_LOCK(curthread, pthread);
+		pthread->flags |= THR_FLAGS_DETACHED;
 		pthread->joiner = NULL;
-		THR_GCLIST_ADD(pthread);
-		THREAD_LIST_UNLOCK(curthread);
+		_thr_try_gc(curthread, pthread); /* thread lock released */
 
 		if (thread_return != NULL)
 			*thread_return = tmp;

lib/libthr/thread/thr_kill.c

@@ -54,11 +54,15 @@ _pthread_kill(pthread_t pthread, int sig)
 	 * signal is valid (signal 0 specifies error checking only) and
 	 * not being ignored:
 	 */
-	else if ((ret = _thr_ref_add(curthread, pthread, /*include dead*/0))
+	else if (curthread == pthread) {
+		if (sig > 0)
+			_thr_send_sig(pthread, sig);
+		ret = 0;
+	} if ((ret = _thr_find_thread(curthread, pthread, /*include dead*/0))
 	    == 0) {
 		if (sig > 0)
 			_thr_send_sig(pthread, sig);
-		_thr_ref_delete(curthread, pthread);
+		THR_THREAD_UNLOCK(curthread, pthread);
 	}
 
 	/* Return the completion status: */