Implement independent CPU preemptions

rpcs3/Emu/CPU/CPUThread.cpp

@@ -54,6 +54,8 @@ void fmt_class_string<cpu_flag>::format(std::string& out, u64 arg)
 		case cpu_flag::memory: return "mem";
 		case cpu_flag::pending: return "pend";
 		case cpu_flag::pending_recheck: return "pend-re";
+		case cpu_flag::yield: return "y";
+		case cpu_flag::preempt: return "PREEMPT";
 		case cpu_flag::dbg_global_pause: return "G-PAUSE";
 		case cpu_flag::dbg_pause: return "PAUSE";
 		case cpu_flag::dbg_step: return "STEP";
@@ -574,6 +576,7 @@ void cpu_thread::operator()()
 		if (!(state0 & cpu_flag::stop))
 		{
 			cpu_task();
+			state += cpu_flag::wait;
 
 			if (state & cpu_flag::ret && state.test_and_reset(cpu_flag::ret))
 			{
@@ -730,14 +733,20 @@ bool cpu_thread::check_state() noexcept
 			if (!is_stopped(flags) && flags.none_of(cpu_flag::ret))
 			{
 				// Check pause flags which hold thread inside check_state (ignore suspend/debug flags on cpu_flag::temp)
-				if (flags & (cpu_flag::pause + cpu_flag::memory) || (cpu_can_stop && flags & (cpu_flag::dbg_global_pause + cpu_flag::dbg_pause + cpu_flag::suspend)))
+				if (flags & (cpu_flag::pause + cpu_flag::memory + cpu_flag::yield + cpu_flag::preempt) || (cpu_can_stop && flags & (cpu_flag::dbg_global_pause + cpu_flag::dbg_pause + cpu_flag::suspend)))
 				{
 					if (!(flags & cpu_flag::wait))
 					{
 						flags += cpu_flag::wait;
 						store = true;
 					}
 
+					if (flags & (cpu_flag::yield + cpu_flag::preempt))
+					{
+						flags -= (cpu_flag::yield + cpu_flag::preempt);
+						store = true;
+					}
+
 					escape = false;
 					state1 = flags;
 					return store;
@@ -767,6 +776,30 @@ bool cpu_thread::check_state() noexcept
 			return store;
 		}).first;
 
+		if (state0 & cpu_flag::preempt)
+		{
+			if (cpu_flag::wait - state0)
+			{
+				// Yield itself
+				state.wait(state1, atomic_wait_timeout{20'000});
+			}
+
+			if (const u128 bits = s_cpu_bits)
+			{
+				reader_lock lock(s_cpu_lock);
+
+				cpu_counter::for_all_cpu(bits & s_cpu_bits, [](cpu_thread* cpu)
+				{
+					if (cpu->state.none_of(cpu_flag::wait + cpu_flag::yield))
+					{
+						cpu->state += cpu_flag::yield;
+					}
+
+					return true;
+				});
+			}
+		}
+
 		if (escape)
 		{
 			if (s_tls_thread_slot == umax && !retval)
@@ -855,6 +888,14 @@ bool cpu_thread::check_state() noexcept
 						break;
 					}
 				}
+
+				continue;
+			}
+
+			if (state0 & cpu_flag::yield && cpu_flag::wait - state0)
+			{
+				// Short sleep when yield flag is present alone (makes no sense when other methods which can stop thread execution have been done)
+				state.wait(state1, atomic_wait_timeout{20'000});
 			}
 		}
 	}

rpcs3/Emu/CPU/CPUThread.h

@@ -25,6 +25,8 @@ enum class cpu_flag : u32
 	pending, // Thread has postponed work
 	pending_recheck, // Thread needs to recheck if there is pending work before ::pending removal
 	notify, // Flag meant solely to allow atomic notification on state without changing other flags
+	yield, // Thread is being requested to yield its execution time if it's running
+	preempt, // Thread is being requested to preempt the execution of all CPU threads
 
 	dbg_global_pause, // Emulation paused
 	dbg_pause, // Thread paused

rpcs3/Emu/Cell/SPUThread.cpp

@@ -4225,7 +4225,7 @@ s64 spu_thread::get_ch_value(u32 ch)
 
 		spu_function_logger logger(*this, "MFC Events read");
 
-		state += cpu_flag::wait;
+		lv2_obj::prepare_for_sleep(*this);
 
 		using resrv_ptr = std::add_pointer_t<const decltype(rdata)>;
 
@@ -4913,7 +4913,7 @@ bool spu_thread::stop_and_signal(u32 code)
 			return ch_in_mbox.set_values(1, CELL_EINVAL), true;
 		}
 
-		state += cpu_flag::wait;
+		lv2_obj::prepare_for_sleep(*this);
 
 		spu_function_logger logger(*this, "sys_spu_thread_receive_event");
 

rpcs3/Emu/Cell/lv2/lv2.cpp

@@ -51,9 +51,15 @@
 
 #include <optional>
 #include <deque>
+#include "util/tsc.hpp"
 
 extern std::string ppu_get_syscall_name(u64 code);
 
+namespace rsx
+{
+	void set_rsx_yield_flag() noexcept;
+}
+
 template <>
 void fmt_class_string<ppu_syscall_code>::format(std::string& out, u64 arg)
 {
@@ -1202,6 +1208,10 @@ static std::deque<std::pair<u64, class cpu_thread*>> g_waiting;
 // Threads which must call lv2_obj::sleep before the scheduler starts
 static std::deque<class cpu_thread*> g_to_sleep;
 
+static atomic_t<u64> s_yield_frequency = 0;
+static atomic_t<u64> s_max_allowed_yield_tsc = 0;
+static u64 s_last_yield_tsc = 0;
+
 namespace cpu_counter
 {
 	void remove(cpu_thread*) noexcept;
@@ -1577,6 +1587,7 @@ void lv2_obj::cleanup()
 	g_to_sleep.clear();
 	g_waiting.clear();
 	g_pending = 0;
+	s_yield_frequency = 0;
 }
 
 void lv2_obj::schedule_all(u64 current_time)
@@ -1653,6 +1664,22 @@ void lv2_obj::schedule_all(u64 current_time)
 		// Null-terminate the list if it ends before last slot
 		g_to_notify[notify_later_idx] = nullptr;
 	}
+
+	if (const u64 freq = s_yield_frequency)
+	{
+		if (auto cpu = cpu_thread::get_current())
+		{
+			const u64 tsc = utils::get_tsc();
+			const u64 last_tsc = s_last_yield_tsc;
+
+			if (tsc >= last_tsc && tsc <= s_max_allowed_yield_tsc && tsc - last_tsc >= freq)
+			{
+				cpu->state += cpu_flag::preempt;
+				s_last_yield_tsc = tsc;
+				rsx::set_rsx_yield_flag();
+			}
+		}
+	}
 }
 
 ppu_thread_status lv2_obj::ppu_state(ppu_thread* ppu, bool lock_idm, bool lock_lv2)
@@ -1737,6 +1764,12 @@ bool lv2_obj::has_ppus_in_running_state()
 	return false;
 }
 
+void lv2_obj::set_yield_frequency(u64 freq, u64 max_allowed_tsc)
+{
+	s_yield_frequency.release(freq);
+	s_max_allowed_yield_tsc.release(max_allowed_tsc);
+}
+
 bool lv2_obj::wait_timeout(u64 usec, ppu_thread* cpu, bool scale, bool is_usleep)
 {
 	static_assert(u64{umax} / max_timeout >= 100, "max timeout is not valid for scaling");

rpcs3/Emu/Cell/lv2/sys_sync.h

@@ -266,6 +266,8 @@ struct lv2_obj
 	// Must be called under IDM lock
 	static bool has_ppus_in_running_state();
 
+	static void set_yield_frequency(u64 freq, u64 max_allowed_tsx);
+
 	static void cleanup();
 
 	template <typename T>