vk: Handle VK_ERROR_FRAGMENTATION when allocating descriptor pools

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -1202,6 +1202,19 @@ bool VKGSRender::on_vram_exhausted(rsx::problem_severity severity)
 	return any_cache_relieved;
 }
 
+void VKGSRender::on_descriptor_pool_fragmentation(bool is_fatal)
+{
+	if (!is_fatal)
+	{
+		// It is very likely that the release is simply in progress (enqueued)
+		m_primary_cb_list.wait_all();
+		return;
+	}
+
+	// Just flush everything. Unless the hardware is very deficient, this should happen very rarely.
+	flush_command_queue(true, true);
+}
+
 void VKGSRender::notify_tile_unbound(u32 tile)
 {
 	//TODO: Handle texture writeback

rpcs3/Emu/RSX/VK/VKGSRender.h

@@ -263,6 +263,9 @@ class VKGSRender : public GSRender, public ::rsx::reports::ZCULL_control
 	// External callback to handle out of video memory problems
 	bool on_vram_exhausted(rsx::problem_severity severity);
 
+	// Handle pool creation failure due to fragmentation
+	void on_descriptor_pool_fragmentation(bool is_fatal);
+
 	// Conditional rendering
 	void begin_conditional_rendering(const std::vector<rsx::reports::occlusion_query_info*>& sources) override;
 	void end_conditional_rendering() override;

rpcs3/Emu/RSX/VK/VKGSRenderTypes.hpp

@@ -345,6 +345,14 @@ namespace vk
 			}
 		}
 
+		void wait_all()
+		{
+			for (auto& cb : m_cb_list)
+			{
+				cb.wait();
+			}
+		}
+
 		inline command_buffer_chunk* next()
 		{
 			const auto result_id = ++m_current_index % Count;

rpcs3/Emu/RSX/VK/VKHelpers.cpp

@@ -268,4 +268,12 @@ namespace vk
 
 		renderer->emergency_query_cleanup(&cmd);
 	}
+
+	void on_descriptor_pool_fragmentation(bool is_fatal)
+	{
+		if (auto vkthr = dynamic_cast<VKGSRender*>(rsx::get_current_renderer()))
+		{
+			vkthr->on_descriptor_pool_fragmentation(is_fatal);
+		}
+	}
 }

rpcs3/Emu/RSX/VK/vkutils/descriptors.cpp

@@ -4,6 +4,9 @@
 
 namespace vk
 {
+	// Error handler callback
+	extern void on_descriptor_pool_fragmentation(bool fatal);
+
 	namespace descriptors
 	{
 		class dispatch_manager
@@ -111,6 +114,7 @@ namespace vk
 		ensure(max_sets > 16);
 
 		m_create_info_pool_sizes = pool_sizes;
+
 		for (auto& size : m_create_info_pool_sizes)
 		{
 			ensure(size.descriptorCount < 128); // Sanity check. Remove before commit.
@@ -221,24 +225,39 @@ namespace vk
 			vk::get_gc()->dispose(cleanup_obj);
 		}
 
-		std::lock_guard lock(m_subpool_lock);
-
 		m_current_subpool_offset = 0;
 		m_current_subpool_index = umax;
 
-		for (u32 index = 0; index < m_device_subpools.size(); ++index)
+		const int max_retries = 2;
+		int retries = max_retries;
+
+		do
 		{
-			if (!m_device_subpools[index].busy)
+			for (u32 index = 0; index < m_device_subpools.size(); ++index)
 			{
-				m_current_subpool_index = index;
-				break;
+				if (!m_device_subpools[index].busy)
+				{
+					m_current_subpool_index = index;
+					goto done; // Nested break
+				}
 			}
-		}
 
-		if (m_current_subpool_index == umax)
-		{
 			VkDescriptorPool subpool = VK_NULL_HANDLE;
-			CHECK_RESULT(vkCreateDescriptorPool(*m_owner, &m_create_info, nullptr, &subpool));
+			if (VkResult result = vkCreateDescriptorPool(*m_owner, &m_create_info, nullptr, &subpool))
+			{
+				if (retries-- && (result == VK_ERROR_FRAGMENTATION_EXT))
+				{
+					rsx_log.warning("Descriptor pool creation failed with fragmentation error. Will attempt to recover.");
+					vk::on_descriptor_pool_fragmentation(!retries);
+					continue;
+				}
+
+				vk::die_with_error(result);
+				fmt::throw_exception("Unreachable");
+			}
+
+			// New subpool created successfully
+			std::lock_guard lock(m_subpool_lock);
 
 			m_device_subpools.push_back(
 			{
@@ -247,8 +266,10 @@ namespace vk
 			});
 
 			m_current_subpool_index = m_device_subpools.size() - 1;
-		}
 
+		} while (m_current_subpool_index == umax);
+
+	done:
 		m_device_subpools[m_current_subpool_index].busy = VK_TRUE;
 		m_current_pool_handle = m_device_subpools[m_current_subpool_index].handle;
 	}

rpcs3/Emu/RSX/VK/vkutils/shared.cpp

@@ -96,6 +96,9 @@ namespace vk
 		case VK_ERROR_INVALID_EXTERNAL_HANDLE_KHR:
 			error_message = "Invalid external handle (VK_ERROR_INVALID_EXTERNAL_HANDLE_KHR)";
 			break;
+		case VK_ERROR_FRAGMENTATION_EXT:
+			error_message = "Descriptor pool creation failed (VK_ERROR_FRAGMENTATION)";
+			break;
 		default:
 			error_message = fmt::format("Unknown Code (%Xh, %d)%s", static_cast<s32>(error_code), static_cast<s32>(error_code), src_loc{line, col, file, func});
 			break;