Merge pull request #18328 from hrydgard/desc-on-render-thread-2

Vulkan: Write descriptor sets on the render thread

Common/Data/Collections/FastVec.h

@@ -69,6 +69,7 @@ class FastVec {
 	void clear() { size_ = 0; }
 	bool empty() const { return size_ == 0; }
 
+	const T *data() { return data_; }
 	T *begin() { return data_; }
 	T *end() { return data_ + size_; }
 	const T *begin() const { return data_; }
@@ -117,6 +118,12 @@ class FastVec {
 		IncreaseCapacityTo(newCapacity);
 	}
 
+	void extend(const T *newData, size_t count) {
+		IncreaseCapacityTo(size_ + count);
+		memcpy(data_ + size_, newData, count * sizeof(T));
+		size_ += count;
+	}
+
 	void LockCapacity() {
 #ifdef _DEBUG
 		capacityLocked_ = true;

Common/GPU/Vulkan/VulkanDescSet.cpp

@@ -42,21 +42,21 @@ void VulkanDescSetPool::Create(VulkanContext *vulkan, const BindingType *binding
 	_assert_msg_(res == VK_SUCCESS, "Could not create VulkanDescSetPool %s", tag_);
 }
 
-VkDescriptorSet VulkanDescSetPool::Allocate(int n, const VkDescriptorSetLayout *layouts, const char *tag) {
-	if (descPool_ == VK_NULL_HANDLE || usage_ + n >= info_.maxSets) {
+bool VulkanDescSetPool::Allocate(VkDescriptorSet *descriptorSets, int count, const VkDescriptorSetLayout *layouts) {
+	if (descPool_ == VK_NULL_HANDLE || usage_ + count >= info_.maxSets) {
 		// Missing or out of space, need to recreate.
 		VkResult res = Recreate(grow_);
 		_assert_msg_(res == VK_SUCCESS, "Could not grow VulkanDescSetPool %s on usage %d", tag_, (int)usage_);
 	}
 
-	VkDescriptorSet desc;
 	VkDescriptorSetAllocateInfo descAlloc{ VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO };
 	descAlloc.descriptorPool = descPool_;
-	descAlloc.descriptorSetCount = n;
+	descAlloc.descriptorSetCount = count;
 	descAlloc.pSetLayouts = layouts;
-	VkResult result = vkAllocateDescriptorSets(vulkan_->GetDevice(), &descAlloc, &desc);
+	VkResult result = vkAllocateDescriptorSets(vulkan_->GetDevice(), &descAlloc, descriptorSets);
 
 	if (result == VK_ERROR_FRAGMENTED_POOL || result < 0) {
+		WARN_LOG(G3D, "Pool %s %s - recreating", tag_, result == VK_ERROR_FRAGMENTED_POOL ? "fragmented" : "full");
 		// There seems to have been a spec revision. Here we should apparently recreate the descriptor pool,
 		// so let's do that. See https://www.khronos.org/registry/vulkan/specs/1.0/man/html/vkAllocateDescriptorSets.html
 		// Fragmentation shouldn't really happen though since we wipe the pool every frame.
@@ -65,18 +65,16 @@ VkDescriptorSet VulkanDescSetPool::Allocate(int n, const VkDescriptorSetLayout *
 
 		// Need to update this pointer since we have allocated a new one.
 		descAlloc.descriptorPool = descPool_;
-		result = vkAllocateDescriptorSets(vulkan_->GetDevice(), &descAlloc, &desc);
+		result = vkAllocateDescriptorSets(vulkan_->GetDevice(), &descAlloc, descriptorSets);
 		_assert_msg_(result == VK_SUCCESS, "Ran out of descriptor space (frag?) and failed to allocate after recreating a descriptor pool. res=%d", (int)result);
 	}
 
 	if (result != VK_SUCCESS) {
-		return VK_NULL_HANDLE;
+		return false;
 	}
 
-	usage_++;
-
-	vulkan_->SetDebugName(desc, VK_OBJECT_TYPE_DESCRIPTOR_SET, tag);
-	return desc;
+	usage_ += count;
+	return true;
 }
 
 void VulkanDescSetPool::Reset() {
@@ -110,7 +108,7 @@ VkResult VulkanDescSetPool::Recreate(bool grow) {
 
 	// Delete the pool if it already exists.
 	if (descPool_ != VK_NULL_HANDLE) {
-		DEBUG_LOG(G3D, "Reallocating %s desc pool from %d to %d", tag_, prevSize, info_.maxSets);
+		INFO_LOG(G3D, "Reallocating %s desc pool from %d to %d", tag_, prevSize, info_.maxSets);
 		vulkan_->Delete().QueueDeleteDescriptorPool(descPool_);
 		clear_();
 		usage_ = 0;

Common/GPU/Vulkan/VulkanDescSet.h

@@ -18,7 +18,7 @@ enum class BindingType {
 // Only appropriate for use in a per-frame pool.
 class VulkanDescSetPool {
 public:
-	VulkanDescSetPool(const char *tag, bool grow) : tag_(tag), grow_(grow) {}
+	VulkanDescSetPool(const char *tag = "", bool grow = true) : tag_(tag), grow_(grow) {}
 	~VulkanDescSetPool();
 
 	// Must call this before use: defines how to clear cache of ANY returned values from Allocate().
@@ -28,10 +28,17 @@ class VulkanDescSetPool {
 	void Create(VulkanContext *vulkan, const BindingType *bindingTypes, uint32_t bindingTypesCount, uint32_t descriptorCount);
 	// Allocate a new set, which may resize and empty the current sets.
 	// Use only for the current frame, unless in a cache cleared by clear_.
-	VkDescriptorSet Allocate(int n, const VkDescriptorSetLayout *layouts, const char *tag);
+	bool Allocate(VkDescriptorSet *descriptorSets, int count, const VkDescriptorSetLayout *layouts);
 	void Reset();
 	void Destroy();
 
+	void SetTag(const char *tag) {
+		tag_ = tag;
+	}
+	bool IsDestroyed() const {
+		return !descPool_;
+	}
+
 private:
 	VkResult Recreate(bool grow);
 

Common/GPU/Vulkan/VulkanFrameData.h

@@ -28,6 +28,7 @@ struct QueueProfileContext {
 	double cpuStartTime;
 	double cpuEndTime;
 	double descWriteTime;
+	int descriptorsWritten;
 };
 
 class VKRFramebuffer;

Common/GPU/Vulkan/VulkanQueueRunner.cpp

@@ -338,7 +338,7 @@ void VulkanQueueRunner::PreprocessSteps(std::vector<VKRStep *> &steps) {
 	}
 }
 
-void VulkanQueueRunner::RunSteps(std::vector<VKRStep *> &steps, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps) {
+void VulkanQueueRunner::RunSteps(std::vector<VKRStep *> &steps, int curFrame, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps) {
 	QueueProfileContext *profile = frameData.profile.enabled ? &frameData.profile : nullptr;
 
 	if (profile)
@@ -394,7 +394,7 @@ void VulkanQueueRunner::RunSteps(std::vector<VKRStep *> &steps, FrameData &frame
 					vkCmdBeginDebugUtilsLabelEXT(cmd, &labelInfo);
 				}
 			}
-			PerformRenderPass(step, cmd);
+			PerformRenderPass(step, cmd, curFrame);
 			break;
 		case VKRStepType::COPY:
 			PerformCopy(step, cmd);
@@ -1103,7 +1103,7 @@ void TransitionFromOptimal(VkCommandBuffer cmd, VkImage colorImage, VkImageLayou
 	}
 }
 
-void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer cmd) {
+void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer cmd, int curFrame) {
 	for (size_t i = 0; i < step.preTransitions.size(); i++) {
 		const TransitionRequest &iter = step.preTransitions[i];
 		if (iter.aspect == VK_IMAGE_ASPECT_COLOR_BIT && iter.fb->color.layout != iter.targetLayout) {
@@ -1199,6 +1199,7 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 	// The stencil ones are very commonly mostly redundant so let's eliminate them where possible.
 	// Might also want to consider scissor and viewport.
 	VkPipeline lastPipeline = VK_NULL_HANDLE;
+	FastVec<PendingDescSet> *descSets = nullptr;
 	VkPipelineLayout pipelineLayout = VK_NULL_HANDLE;
 
 	bool pipelineOK = false;
@@ -1241,6 +1242,7 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 
 				if (pipeline != VK_NULL_HANDLE) {
 					vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
+					descSets = &c.pipeline.pipelineLayout->frameData[curFrame].descSets_;
 					pipelineLayout = c.pipeline.pipelineLayout->pipelineLayout;
 					lastGraphicsPipeline = graphicsPipeline;
 					pipelineOK = true;
@@ -1336,7 +1338,8 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 
 		case VKRRenderCommand::DRAW_INDEXED:
 			if (pipelineOK) {
-				vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &c.drawIndexed.ds, c.drawIndexed.numUboOffsets, c.drawIndexed.uboOffsets);
+				VkDescriptorSet set = (*descSets)[c.drawIndexed.descSetIndex].set;
+				vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &set, c.drawIndexed.numUboOffsets, c.drawIndexed.uboOffsets);
 				vkCmdBindIndexBuffer(cmd, c.drawIndexed.ibuffer, c.drawIndexed.ioffset, VK_INDEX_TYPE_UINT16);
 				VkDeviceSize voffset = c.drawIndexed.voffset;
 				vkCmdBindVertexBuffers(cmd, 0, 1, &c.drawIndexed.vbuffer, &voffset);
@@ -1346,7 +1349,9 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 
 		case VKRRenderCommand::DRAW:
 			if (pipelineOK) {
-				vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &c.draw.ds, c.draw.numUboOffsets, c.draw.uboOffsets);
+				VkDescriptorSet set = (*descSets)[c.drawIndexed.descSetIndex].set;
+				_dbg_assert_(set != VK_NULL_HANDLE);
+				vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &set, c.draw.numUboOffsets, c.draw.uboOffsets);
 				if (c.draw.vbuffer) {
 					vkCmdBindVertexBuffers(cmd, 0, 1, &c.draw.vbuffer, &c.draw.voffset);
 				}

Common/GPU/Vulkan/VulkanQueueRunner.h

@@ -70,7 +70,7 @@ struct VkRenderData {
 			VKRPipelineLayout *pipelineLayout;
 		} compute_pipeline;
 		struct {
-			VkDescriptorSet ds;
+			uint32_t descSetIndex;
 			int numUboOffsets;
 			uint32_t uboOffsets[3];
 			VkBuffer vbuffer;
@@ -79,7 +79,7 @@ struct VkRenderData {
 			uint32_t offset;
 		} draw;
 		struct {
-			VkDescriptorSet ds;
+			uint32_t descSetIndex;
 			uint32_t uboOffsets[3];
 			uint16_t numUboOffsets;
 			uint16_t instances;
@@ -119,9 +119,7 @@ struct VkRenderData {
 			const char *annotation;
 		} debugAnnotation;
 		struct {
-			int setNumber;
-			VkDescriptorSet set;
-			VkPipelineLayout pipelineLayout;
+			int setIndex;
 		} bindDescSet;
 	};
 };
@@ -231,7 +229,7 @@ class VulkanQueueRunner {
 	}
 
 	void PreprocessSteps(std::vector<VKRStep *> &steps);
-	void RunSteps(std::vector<VKRStep *> &steps, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps = false);
+	void RunSteps(std::vector<VKRStep *> &steps, int curFrame, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps = false);
 	void LogSteps(const std::vector<VKRStep *> &steps, bool verbose);
 
 	static std::string StepToString(VulkanContext *vulkan, const VKRStep &step);
@@ -291,7 +289,7 @@ class VulkanQueueRunner {
 	bool InitDepthStencilBuffer(VkCommandBuffer cmd);  // Used for non-buffered rendering.
 
 	VKRRenderPass *PerformBindFramebufferAsRenderTarget(const VKRStep &pass, VkCommandBuffer cmd);
-	void PerformRenderPass(const VKRStep &pass, VkCommandBuffer cmd);
+	void PerformRenderPass(const VKRStep &pass, VkCommandBuffer cmd, int curFrame);
 	void PerformCopy(const VKRStep &pass, VkCommandBuffer cmd);
 	void PerformBlit(const VKRStep &pass, VkCommandBuffer cmd);
 	void PerformReadback(const VKRStep &pass, VkCommandBuffer cmd, FrameData &frameData);