Vulkan: Implement basic integrated GPU profiling.

Core/Config.cpp

@@ -962,6 +962,7 @@ static ConfigSetting debuggerSettings[] = {
 	ConfigSetting("ShowBottomTabTitles",&g_Config.bShowBottomTabTitles,true),
 	ConfigSetting("ShowDeveloperMenu", &g_Config.bShowDeveloperMenu, false),
 	ConfigSetting("ShowAllocatorDebug", &g_Config.bShowAllocatorDebug, false, false),
+	ConfigSetting("ShowGpuProfile", &g_Config.bShowGpuProfile, false, false),
 	ConfigSetting("SkipDeadbeefFilling", &g_Config.bSkipDeadbeefFilling, false),
 	ConfigSetting("FuncHashMap", &g_Config.bFuncHashMap, false),
 

Core/Config.h

@@ -238,6 +238,7 @@ struct Config {
 	bool bShowDebugStats;
 	bool bShowAudioDebug;
 	bool bAudioResampler;
+	bool bShowGpuProfile;
 
 	//Analog stick tilting
 	//the base x and y tilt. this inclination is treated as (0,0) and the tilt input

GPU/Vulkan/DebugVisVulkan.cpp

@@ -26,6 +26,8 @@
 #include "GPU/Vulkan/GPU_Vulkan.h"
 #include "GPU/Vulkan/VulkanUtil.h"
 
+#undef DrawText
+
 void DrawAllocatorVis(UIContext *ui, GPUInterface *gpu) {
 	if (!gpu) {
 		return;
@@ -94,3 +96,26 @@ void DrawAllocatorVis(UIContext *ui, GPUInterface *gpu) {
 	for (auto iter : texturesToDelete)
 		iter->Release();
 }
+
+void DrawProfilerVis(UIContext *ui, GPUInterface *gpu) {
+	if (!gpu) {
+		return;
+	}
+	using namespace Draw;
+	const int padding = 10;
+	const int columnWidth = 256;
+	const int starty = padding * 8;
+	int x = padding;
+	int y = starty;
+	int w = columnWidth;  // We will double this when actually drawing to make the pixels visible.
+
+	ui->Begin();
+
+	GPU_Vulkan *gpuVulkan = static_cast<GPU_Vulkan *>(gpu);
+
+	std::string text = gpuVulkan->GetGpuProfileString();
+
+	Draw::DrawContext *draw = ui->GetDrawContext();
+	ui->DrawTextShadow(text.c_str(), 10, 50, 0xFFFFFFFF, FLAG_DYNAMIC_ASCII);
+	ui->Flush();
+}

GPU/Vulkan/DebugVisVulkan.h

@@ -24,3 +24,4 @@ class UIContext;
 
 // gpu MUST be an instance of GPU_Vulkan. If not, will definitely crash.
 void DrawAllocatorVis(UIContext *ui, GPUInterface *gpu);
+void DrawProfilerVis(UIContext *ui, GPUInterface *gpu);

GPU/Vulkan/GPU_Vulkan.cpp

@@ -638,3 +638,8 @@ std::string GPU_Vulkan::DebugGetShaderString(std::string id, DebugShaderType typ
 		return std::string();
 	}
 }
+
+std::string GPU_Vulkan::GetGpuProfileString() {
+	VulkanRenderManager *rm = (VulkanRenderManager *)draw_->GetNativeObject(Draw::NativeObject::RENDER_MANAGER);
+	return rm->GetGpuProfileString();
+}

GPU/Vulkan/GPU_Vulkan.h

@@ -66,6 +66,8 @@ class GPU_Vulkan : public GPUCommon {
 		return textureCacheVulkan_;
 	}
 
+	std::string GetGpuProfileString();
+
 protected:
 	void FinishDeferred() override;
 

UI/DevScreens.cpp

@@ -78,7 +78,10 @@ void DevMenu::CreatePopupContents(UI::ViewGroup *parent) {
 	items->Add(new Choice(sy->T("Developer Tools")))->OnClick.Handle(this, &DevMenu::OnDeveloperTools);
 	items->Add(new Choice(dev->T("Jit Compare")))->OnClick.Handle(this, &DevMenu::OnJitCompare);
 	items->Add(new Choice(dev->T("Shader Viewer")))->OnClick.Handle(this, &DevMenu::OnShaderView);
-	items->Add(new CheckBox(&g_Config.bShowAllocatorDebug, dev->T("Allocator Viewer")));
+	if (g_Config.iGPUBackend == (int)GPUBackend::VULKAN) {
+		items->Add(new CheckBox(&g_Config.bShowAllocatorDebug, dev->T("Allocator Viewer")));
+		items->Add(new CheckBox(&g_Config.bShowGpuProfile, dev->T("GPU Profile")));
+	}
 	items->Add(new Choice(dev->T("Toggle Freeze")))->OnClick.Handle(this, &DevMenu::OnFreezeFrame);
 	items->Add(new Choice(dev->T("Dump Frame GPU Commands")))->OnClick.Handle(this, &DevMenu::OnDumpFrame);
 	items->Add(new Choice(dev->T("Toggle Audio Debug")))->OnClick.Handle(this, &DevMenu::OnToggleAudioDebug);

UI/EmuScreen.cpp

@@ -1359,6 +1359,11 @@ void EmuScreen::renderUI() {
 	if (g_Config.iGPUBackend == (int)GPUBackend::VULKAN && g_Config.bShowAllocatorDebug) {
 		DrawAllocatorVis(ctx, gpu);
 	}
+
+	if (g_Config.iGPUBackend == (int)GPUBackend::VULKAN && g_Config.bShowGpuProfile) {
+		DrawProfilerVis(ctx, gpu);
+	}
+
 #endif
 
 #ifdef USE_PROFILER

ext/native/thin3d/VulkanRenderManager.cpp

@@ -1,6 +1,8 @@
 #include <algorithm>
 #include <cstdint>
 
+#include <sstream>
+
 #include "Common/Log.h"
 #include "base/logging.h"
 
@@ -131,6 +133,11 @@ VulkanRenderManager::VulkanRenderManager(VulkanContext *vulkan) : vulkan_(vulkan
 		res = vkAllocateCommandBuffers(vulkan_->GetDevice(), &cmd_alloc, &frameData_[i].mainCmd);
 		assert(res == VK_SUCCESS);
 		frameData_[i].fence = vulkan_->CreateFence(true);  // So it can be instantly waited on
+
+		VkQueryPoolCreateInfo query_ci{ VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO };
+		query_ci.queryCount = MAX_TIMESTAMP_QUERIES;
+		query_ci.queryType = VK_QUERY_TYPE_TIMESTAMP;
+		res = vkCreateQueryPool(vulkan_->GetDevice(), &query_ci, nullptr, &frameData_[i].timestampQueryPool_);
 	}
 
 	queueRunner_.CreateDeviceObjects();
@@ -289,6 +296,7 @@ VulkanRenderManager::~VulkanRenderManager() {
 		vkDestroyCommandPool(device, frameData_[i].cmdPoolInit, nullptr);
 		vkDestroyCommandPool(device, frameData_[i].cmdPoolMain, nullptr);
 		vkDestroyFence(device, frameData_[i].fence, nullptr);
+		vkDestroyQueryPool(device, frameData_[i].timestampQueryPool_, nullptr);
 	}
 	queueRunner_.DestroyDeviceObjects();
 }
@@ -360,6 +368,38 @@ void VulkanRenderManager::BeginFrame() {
 	vkWaitForFences(device, 1, &frameData.fence, true, UINT64_MAX);
 	vkResetFences(device, 1, &frameData.fence);
 
+	uint64_t queryResults[MAX_TIMESTAMP_QUERIES];
+	if (gpuProfilingEnabled_) {
+		// Pull the profiling results from last time and produce a summary!
+		if (frameData.numQueries) {
+			VkResult res = vkGetQueryPoolResults(
+				vulkan_->GetDevice(),
+				frameData.timestampQueryPool_, 0, frameData.numQueries, sizeof(uint64_t) * frameData.numQueries, &queryResults[0], sizeof(uint64_t),
+				VK_QUERY_RESULT_64_BIT);
+			if (res == VK_SUCCESS) {
+				double timestampConversionFactor = (double)vulkan_->GetPhysicalDeviceProperties().properties.limits.timestampPeriod * (1.0 / 1000000.0);
+				uint64_t timestampDiffMask = 0xFFFFFFFFFFFFFFFFULL;  // TODO: Get from queue family
+				std::stringstream str;
+
+				char line[256];
+				snprintf(line, sizeof(line), "Total GPU time: %0.3f ms\n", ((double)((queryResults[frameData.numQueries - 1] - queryResults[0]) & timestampDiffMask) * timestampConversionFactor));
+				str << line;
+				for (int i = 0; i < frameData.numQueries - 1; i++) {
+					uint64_t diff = (queryResults[i + 1] - queryResults[i]) & timestampDiffMask;
+					double milliseconds = (double)diff * timestampConversionFactor;
+					snprintf(line, sizeof(line), "%s: %0.3f ms\n", frameData.timestampDescriptions[i + 1].c_str(), milliseconds);
+					str << line;
+				}
+				frameData.profileSummary = str.str();
+			} else {
+				frameData.profileSummary = "(error getting GPU profile - not ready?)";
+			}
+		}
+		else {
+			frameData.profileSummary = "(no GPU profile data collected)";
+		}
+	}
+
 	// Must be after the fence - this performs deletes.
 	VLOG("PUSH: BeginFrame %d", curFrame);
 	if (!run_) {
@@ -368,6 +408,20 @@ void VulkanRenderManager::BeginFrame() {
 	vulkan_->BeginFrame();
 
 	insideFrame_ = true;
+
+	if (gpuProfilingEnabled_) {
+		// For various reasons, we need to always use an init cmd buffer in this case to perform the vkCmdResetQueryPool,
+		// unless we want to limit ourselves to only measure the main cmd buffer.
+		// Reserve the first two queries for initCmd.
+		frameData.numQueries = 2;
+		frameData.timestampDescriptions.push_back("initCmd Begin");
+		frameData.timestampDescriptions.push_back("initCmd");
+		VkCommandBuffer initCmd = GetInitCmd();
+		vkCmdResetQueryPool(initCmd, frameData.timestampQueryPool_, 0, MAX_TIMESTAMP_QUERIES);
+		vkCmdWriteTimestamp(initCmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, frameData.timestampQueryPool_, 0);
+	} else {
+		frameData.numQueries = 0;
+	}
 }
 
 VkCommandBuffer VulkanRenderManager::GetInitCmd() {
@@ -884,6 +938,7 @@ void VulkanRenderManager::BeginSubmitFrame(int frame) {
 		VkCommandBufferBeginInfo begin{ VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO };
 		begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
 		res = vkBeginCommandBuffer(frameData.mainCmd, &begin);
+
 		_assert_msg_(G3D, res == VK_SUCCESS, "vkBeginCommandBuffer failed! result=%s", VulkanResultToString(res));
 
 		queueRunner_.SetBackbuffer(framebuffers_[frameData.curSwapchainImage], swapchainImages_[frameData.curSwapchainImage].image);
@@ -895,10 +950,20 @@ void VulkanRenderManager::BeginSubmitFrame(int frame) {
 void VulkanRenderManager::Submit(int frame, bool triggerFence) {
 	FrameData &frameData = frameData_[frame];
 	if (frameData.hasInitCommands) {
+		if (gpuProfilingEnabled_ && triggerFence) {
+			// Pre-allocated query ID 1.
+			vkCmdWriteTimestamp(frameData.initCmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, frameData.timestampQueryPool_, 1);
+		}
 		VkResult res = vkEndCommandBuffer(frameData.initCmd);
 		_assert_msg_(G3D, res == VK_SUCCESS, "vkEndCommandBuffer failed (init)! result=%s", VulkanResultToString(res));
 	}
 
+	if (gpuProfilingEnabled_) {
+		vkCmdWriteTimestamp(frameData.mainCmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, frameData.timestampQueryPool_, frameData.numQueries);
+		frameData.numQueries++;
+		frameData.timestampDescriptions.push_back("mainCmd");
+	}
+
 	VkResult res = vkEndCommandBuffer(frameData.mainCmd);
 	_assert_msg_(G3D, res == VK_SUCCESS, "vkEndCommandBuffer failed (main)! result=%s", VulkanResultToString(res));
 

ext/native/thin3d/VulkanRenderManager.h

@@ -242,6 +242,15 @@ class VulkanRenderManager {
 		return &queueRunner_;
 	}
 
+	// Call before BeginFrame.
+	void SetGPUProfilingEnabled(bool enabled) {
+		gpuProfilingEnabled_ = enabled;
+	}
+
+	std::string GetGpuProfileString() const {
+		return frameData_[vulkan_->GetCurFrame()].profileSummary;
+	}
+
 private:
 	bool InitBackbufferFramebuffers(int width, int height);
 	bool InitDepthStencilBuffer(VkCommandBuffer cmd);  // Used for non-buffered rendering.
@@ -284,16 +293,28 @@ class VulkanRenderManager {
 		// Swapchain.
 		bool hasBegun = false;
 		uint32_t curSwapchainImage = -1;
+
+		// Profiling.
+		VkQueryPool timestampQueryPool_ = VK_NULL_HANDLE;
+		std::vector<std::string> timestampDescriptions;
+		int numQueries = 0;
+		std::string profileSummary;
 	};
+
 	FrameData frameData_[VulkanContext::MAX_INFLIGHT_FRAMES];
 
+	enum {
+		MAX_TIMESTAMP_QUERIES = 256,
+	};
+
 	// Submission time state
 	int curWidth_ = -1;
 	int curHeight_ = -1;
 	bool insideFrame_ = false;
 	VKRStep *curRenderStep_ = nullptr;
 	std::vector<VKRStep *> steps_;
 	bool splitSubmit_ = false;
+	bool gpuProfilingEnabled_ = true;
 
 	// Execution time state
 	bool run_ = true;

ext/native/thin3d/thin3d_vulkan.cpp

@@ -879,6 +879,7 @@ VKContext::~VKContext() {
 }
 
 void VKContext::BeginFrame() {
+	renderManager_.SetGPUProfilingEnabled(g_Config.bShowGpuProfile);
 	renderManager_.BeginFrame();
 
 	FrameData &frame = frame_[vulkan_->GetCurFrame()];