Implement cpu_thread::suspend_all

Remove Accurate PUTLLC option.
Implement fallback path for SPU transactions.

Utilities/cond.cpp

@@ -273,6 +273,159 @@ void shared_cond::imp_notify() noexcept
 	balanced_awaken<true>(m_cvx32, utils::popcnt32(wait_mask));
 }
 
+void shared_cond::wait_all() noexcept
+{
+	// Try to acquire waiting state without locking but only if there are other locks
+	const auto [old_, result] = m_cvx32.fetch_op([](u64& cvx32) -> u64
+	{
+		// Check waiting alone
+		if ((cvx32 & 0xffffffff) == 0)
+		{
+			return 0;
+		}
+
+		// Combine used bits and invert to find least significant bit unused
+		const u32 slot = utils::cnttz64(~((cvx32 & 0xffffffff) | (cvx32 >> 32)), true);
+
+		// Set waiting bit (does nothing if all slots are used)
+		cvx32 |= (1ull << slot) & 0xffffffff;
+		return 1ull << slot;
+	});
+
+	if (!result)
+	{
+		return;
+	}
+
+	if (result > 0xffffffffu)
+	{
+		// All slots are used, fallback to spin wait
+		while (m_cvx32 & 0xffffffff)
+		{
+			busy_wait();
+		}
+
+		return;
+	}
+
+	const u64 wait_bit = result;
+	const u64 lock_bit = wait_bit | (wait_bit << 32);
+
+	balanced_wait_until(m_cvx32, -1, [&](u64& cvx32, auto... ret) -> int
+	{
+		if ((cvx32 & wait_bit) == 0)
+		{
+			// Remove signal and unlock at once
+			cvx32 &= ~lock_bit;
+			return +1;
+		}
+
+		if constexpr (sizeof...(ret))
+		{
+			cvx32 &= ~lock_bit;
+			return -1;
+		}
+
+		return 0;
+	});
+}
+
+bool shared_cond::wait_all(shared_cond::shared_lock& lock) noexcept
+{
+	AUDIT(lock.m_this == this);
+
+	if (lock.m_slot >= 32)
+	{
+		// Invalid argument, assume notified
+		return true;
+	}
+
+	const u64 wait_bit = c_wait << lock.m_slot;
+	const u64 lock_bit = c_lock << lock.m_slot;
+
+	// Try to acquire waiting state only if there are other locks
+	const auto [old_, not_alone] = m_cvx32.fetch_op([&](u64& cvx32)
+	{
+		// Check locking alone
+		if (((cvx32 >> 32) & cvx32) == (lock_bit >> 32))
+		{
+			return false;
+		}
+
+		// c_lock -> c_wait, c_sig -> unlock
+		cvx32 &= ~(lock_bit & ~wait_bit);
+		return true;
+	});
+
+	if (!not_alone)
+	{
+		return false;
+	}
+	else
+	{
+		// Set invalid slot to acknowledge unlocking
+		lock.m_slot = 33;
+	}
+
+	if ((old_ & wait_bit) == 0)
+	{
+		// Already signaled, return without waiting
+		return true;
+	}
+
+	balanced_wait_until(m_cvx32, -1, [&](u64& cvx32, auto... ret) -> int
+	{
+		if ((cvx32 & wait_bit) == 0)
+		{
+			// Remove signal and unlock at once
+			cvx32 &= ~lock_bit;
+			return +1;
+		}
+
+		if constexpr (sizeof...(ret))
+		{
+			cvx32 &= ~lock_bit;
+			return -1;
+		}
+
+		return 0;
+	});
+
+	return true;
+}
+
+void shared_cond::notify_all(shared_cond::shared_lock& lock) noexcept
+{
+	AUDIT(lock.m_this == this);
+
+	const u64 slot_mask = c_sig << lock.m_slot;
+
+	auto [old, ok] = m_cvx32.fetch_op([&](u64& cvx32)
+	{
+		if (const u64 sig_mask = cvx32 & 0xffffffff)
+		{
+			cvx32 &= (0xffffffffull << 32) & ~slot_mask;
+			cvx32 |= (sig_mask << 32) & ~slot_mask;
+			return true;
+		}
+
+		return false;
+	});
+
+	// Set invalid slot to acknowledge unlocking
+	lock.m_slot = 34;
+
+	// Determine if some waiters need a syscall notification
+	const u64 wait_mask = old & (~old >> 32);
+
+	if (UNLIKELY(!ok || !wait_mask))
+	{
+		return;
+	}
+
+	balanced_awaken<true>(m_cvx32, utils::popcnt32(wait_mask));
+}
+
 bool lf_queue_base::wait(u64 _timeout)
 {
 	auto _old = m_head.compare_and_swap(0, 1);

Utilities/cond.h

@@ -206,7 +206,7 @@ class shared_cond
 			m_slot = m_this->m_cvx32.atomic_op([](u64& cvx32)
 			{
 				// Combine used bits and invert to find least significant bit unused
-				const u32 slot = utils::cnttz32(~((cvx32 & 0xffffffff) | (cvx32 >> 32)), true);
+				const u32 slot = utils::cnttz64(~((cvx32 & 0xffffffff) | (cvx32 >> 32)), true);
 
 				// Set lock bits (does nothing if all slots are used)
 				const u64 bit = (1ull << slot) & 0xffffffff;
@@ -217,6 +217,13 @@ class shared_cond
 
 		shared_lock(const shared_lock&) = delete;
 
+		shared_lock(shared_lock&& rhs)
+			: m_this(rhs.m_this)
+			, m_slot(rhs.m_slot)
+		{
+			rhs.m_slot = 32;
+		}
+
 		shared_lock& operator=(const shared_lock&) = delete;
 
 		~shared_lock()
@@ -261,11 +268,17 @@ class shared_cond
 		return imp_wait(lock.m_slot, usec_timeout);
 	}
 
+	void wait_all() noexcept;
+
+	bool wait_all(shared_lock& lock) noexcept;
+
 	void notify_all() noexcept
 	{
 		if (LIKELY(!m_cvx32))
 			return;
 
 		imp_notify();
 	}
+
+	void notify_all(shared_lock& lock) noexcept;
 };