GPU device choice for D3D11 and Vulkan

Common/Vulkan/VulkanContext.cpp

@@ -213,6 +213,7 @@ VkResult VulkanContext::CreateInstance(const CreateInfo &info) {
 
 	assert(gpu_count > 0);
 	physical_devices_.resize(gpu_count);
+	physicalDeviceProperties_.resize(gpu_count);
 	res = vkEnumeratePhysicalDevices(instance_, &gpu_count, physical_devices_.data());
 	if (res != VK_SUCCESS) {
 		init_error_ = "Failed to enumerate physical devices";
@@ -221,6 +222,9 @@ VkResult VulkanContext::CreateInstance(const CreateInfo &info) {
 		return res;
 	}
 
+	for (uint32_t i = 0; i < gpu_count; i++) {
+		vkGetPhysicalDeviceProperties(physical_devices_[i], &physicalDeviceProperties_[i]);
+	}
 	return VK_SUCCESS;
 }
 
@@ -417,6 +421,14 @@ bool VulkanContext::CheckLayers(const std::vector<LayerProperties> &layer_props,
 	return true;
 }
 
+int VulkanContext::GetPhysicalDeviceByName(std::string name) {
+	for (size_t i = 0; i < physical_devices_.size(); i++) {
+		if (physicalDeviceProperties_[i].deviceName == name)
+			return (int)i;
+	}
+	return -1;
+}
+
 int VulkanContext::GetBestPhysicalDevice() {
 	// Rules: Prefer discrete over embedded.
 	// Prefer nVidia over Intel.
@@ -490,7 +502,6 @@ void VulkanContext::ChooseDevice(int physical_device) {
 
 	// This is as good a place as any to do this
 	vkGetPhysicalDeviceMemoryProperties(physical_devices_[physical_device_], &memory_properties);
-	vkGetPhysicalDeviceProperties(physical_devices_[physical_device_], &gpu_props);
 
 	// Optional features
 	vkGetPhysicalDeviceFeatures(physical_devices_[physical_device_], &featuresAvailable_);

Common/Vulkan/VulkanContext.h

@@ -128,6 +128,7 @@ class VulkanContext {
 	void DestroyInstance();
 
 	int GetBestPhysicalDevice();
+	int GetPhysicalDeviceByName(std::string name);
 	void ChooseDevice(int physical_device);
 	bool EnableDeviceExtension(const char *extension);
 	VkResult CreateDevice();
@@ -173,6 +174,12 @@ class VulkanContext {
 	VkPhysicalDevice GetPhysicalDevice(int n = 0) const {
 		return physical_devices_[n];
 	}
+	int GetCurrentPhysicalDevice() const {
+		return physical_device_;
+	}
+	int GetNumPhysicalDevices() const {
+		return (int)physical_devices_.size();
+	}
 
 	VkQueue GetGraphicsQueue() const {
 		return gfx_queue_;
@@ -182,8 +189,8 @@ class VulkanContext {
 		return graphics_queue_family_index_;
 	}
 
-	const VkPhysicalDeviceProperties &GetPhysicalDeviceProperties() {
-		return gpu_props;
+	const VkPhysicalDeviceProperties &GetPhysicalDeviceProperties(int i) const {
+		return physicalDeviceProperties_[i];
 	}
 
 	VkResult GetInstanceLayerExtensionList(const char *layerName, std::vector<VkExtensionProperties> &extensions);
@@ -281,7 +288,7 @@ class VulkanContext {
 	int physical_device_ = -1;
 
 	uint32_t graphics_queue_family_index_ = -1;
-	VkPhysicalDeviceProperties gpu_props{};
+	std::vector<VkPhysicalDeviceProperties> physicalDeviceProperties_{};
 	std::vector<VkQueueFamilyProperties> queue_props;
 	VkPhysicalDeviceMemoryProperties memory_properties{};
 

Core/Config.cpp

@@ -507,6 +507,10 @@ static ConfigSetting graphicsSettings[] = {
 	ConfigSetting("CardboardYShift", &g_Config.iCardboardXShift, 0, true, true),
 	ConfigSetting("ShowFPSCounter", &g_Config.iShowFPSCounter, 0, true, true),
 	ReportedConfigSetting("GraphicsBackend", &g_Config.iGPUBackend, &DefaultGPUBackend),
+	ConfigSetting("VulkanDevice", &g_Config.sVulkanDevice, "", true, false),
+#ifdef _WIN32
+	ConfigSetting("D3D11Device", &g_Config.sD3D11Device, "", true, false),
+#endif
 	ReportedConfigSetting("RenderingMode", &g_Config.iRenderingMode, &DefaultRenderingMode, true, true),
 	ConfigSetting("SoftwareRenderer", &g_Config.bSoftwareRendering, false, true, true),
 	ReportedConfigSetting("HardwareTransform", &g_Config.bHardwareTransform, true, true, true),

Core/Config.h

@@ -153,6 +153,12 @@ struct Config {
 
 	// GFX
 	int iGPUBackend;
+	// We have separate device parameters for each backend so it doesn't get erased if you switch backends.
+	// If not set, will use the "best" device.
+	std::string sVulkanDevice;
+#ifdef _WIN32
+	std::string sD3D11Device;
+#endif
 	bool bSoftwareRendering;
 	bool bHardwareTransform; // only used in the GLES backend
 	bool bSoftwareSkinning;  // may speed up some games

GPU/Vulkan/DrawEngineVulkan.cpp

@@ -1007,7 +1007,7 @@ void DrawEngineVulkan::TessellationDataTransferVulkan::PrepareBuffers(float *&po
 		float color[4];
 	};
 
-	int ssboAlignment = vulkan_->GetPhysicalDeviceProperties().limits.minStorageBufferOffsetAlignment;
+	int ssboAlignment = vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).limits.minStorageBufferOffsetAlignment;
 	uint8_t *data = (uint8_t *)push_->PushAligned(size * sizeof(TessData), &offset_, &buf_, ssboAlignment);
 	range_ = size * sizeof(TessData);
 

GPU/Vulkan/GPU_Vulkan.cpp

@@ -175,15 +175,15 @@ GPU_Vulkan::~GPU_Vulkan() {
 void GPU_Vulkan::CheckGPUFeatures() {
 	uint32_t features = 0;
 
-	switch (vulkan_->GetPhysicalDeviceProperties().vendorID) {
+	switch (vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).vendorID) {
 	case VULKAN_VENDOR_AMD:
 		// Accurate depth is required on AMD (due to reverse-Z driver bug) so we ignore the compat flag to disable it on those. See #9545
 		features |= GPU_SUPPORTS_ACCURATE_DEPTH;
 		break;
 	case VULKAN_VENDOR_ARM:
 		// Also required on older ARM Mali drivers, like the one on many Galaxy S7.
 		if (!PSP_CoreParameter().compat.flags().DisableAccurateDepth ||
-			  vulkan_->GetPhysicalDeviceProperties().driverVersion <= VK_MAKE_VERSION(428, 811, 2674)) {
+			  vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).driverVersion <= VK_MAKE_VERSION(428, 811, 2674)) {
 			features |= GPU_SUPPORTS_ACCURATE_DEPTH;
 		}
 		break;
@@ -212,15 +212,15 @@ void GPU_Vulkan::CheckGPUFeatures() {
 		features |= GPU_SUPPORTS_DEPTH_CLAMP;
 	}
 	if (vulkan_->GetFeaturesEnabled().dualSrcBlend) {
-		switch (vulkan_->GetPhysicalDeviceProperties().vendorID) {
+		switch (vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).vendorID) {
 		// We thought we had a bug here on nVidia but turns out we accidentally #ifdef-ed out crucial
 		// code on Android.
 		case VULKAN_VENDOR_INTEL:
 			// Workaround for Intel driver bug.
 			break;
 		case VULKAN_VENDOR_AMD:
 			// See issue #10074, and also #10065 (AMD) and #10109 for the choice of the driver version to check for
-			if (vulkan_->GetPhysicalDeviceProperties().driverVersion >= 0x00407000)
+			if (vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).driverVersion >= 0x00407000)
 				features |= GPU_SUPPORTS_DUALSOURCE_BLEND;
 			break;
 		default:
@@ -310,7 +310,7 @@ void GPU_Vulkan::EndHostFrame() {
 
 // Needs to be called on GPU thread, not reporting thread.
 void GPU_Vulkan::BuildReportingInfo() {
-	const auto &props = vulkan_->GetPhysicalDeviceProperties();
+	const auto &props = vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice());
 	const auto &features = vulkan_->GetFeaturesAvailable();
 
 #define CHECK_BOOL_FEATURE(n) do { if (features.n) { featureNames += ", " #n; } } while (false)

GPU/Vulkan/PipelineManagerVulkan.cpp

@@ -676,7 +676,7 @@ bool PipelineManagerVulkan::LoadCache(FILE *file, bool loadRawPipelineCache, Sha
 			WARN_LOG(G3D, "Bad Vulkan pipeline cache header - ignoring");
 			return false;
 		}
-		if (0 != memcmp(header->uuid, vulkan_->GetPhysicalDeviceProperties().pipelineCacheUUID, VK_UUID_SIZE)) {
+		if (0 != memcmp(header->uuid, vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).pipelineCacheUUID, VK_UUID_SIZE)) {
 			// Wrong hardware/driver/etc.
 			WARN_LOG(G3D, "Bad Vulkan pipeline cache UUID - ignoring");
 			return false;

GPU/Vulkan/ShaderManagerVulkan.cpp

@@ -159,7 +159,7 @@ std::string VulkanVertexShader::GetShaderString(DebugShaderStringType type) cons
 ShaderManagerVulkan::ShaderManagerVulkan(VulkanContext *vulkan)
 	: vulkan_(vulkan), lastVShader_(nullptr), lastFShader_(nullptr), fsCache_(16), vsCache_(16) {
 	codeBuffer_ = new char[16384];
-	uboAlignment_ = vulkan_->GetPhysicalDeviceProperties().limits.minUniformBufferOffsetAlignment;
+	uboAlignment_ = vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).limits.minUniformBufferOffsetAlignment;
 	memset(&ub_base, 0, sizeof(ub_base));
 	memset(&ub_lights, 0, sizeof(ub_lights));
 	memset(&ub_bones, 0, sizeof(ub_bones));
@@ -176,7 +176,7 @@ ShaderManagerVulkan::~ShaderManagerVulkan() {
 
 void ShaderManagerVulkan::DeviceRestore(VulkanContext *vulkan) {
 	vulkan_ = vulkan;
-	uboAlignment_ = vulkan_->GetPhysicalDeviceProperties().limits.minUniformBufferOffsetAlignment;
+	uboAlignment_ = vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).limits.minUniformBufferOffsetAlignment;
 }
 
 void ShaderManagerVulkan::Clear() {

GPU/Vulkan/TextureCacheVulkan.cpp

@@ -683,7 +683,7 @@ void TextureCacheVulkan::BuildTexture(TexCacheEntry *const entry) {
 			uint32_t bufferOffset;
 			VkBuffer texBuf;
 			// nvidia returns 1 but that can't be healthy... let's align by 16 as a minimum.
-			int pushAlignment = std::max(16, (int)vulkan_->GetPhysicalDeviceProperties().limits.optimalBufferCopyOffsetAlignment);
+			int pushAlignment = std::max(16, (int)vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).limits.optimalBufferCopyOffsetAlignment);
 			void *data = drawEngine_->GetPushBufferForTextureData()->PushAligned(size, &bufferOffset, &texBuf, pushAlignment);
 			if (replaced.Valid()) {
 				replaced.Load(i, data, stride);

UI/GameSettingsScreen.cpp

@@ -202,6 +202,24 @@ void GameSettingsScreen::CreateViews() {
 		renderingBackendChoice->HideChoice(3);
 	}
 #endif
+	Draw::DrawContext *draw = screenManager()->getDrawContext();
+
+	// Backends that don't allow a device choice will only expose one device.
+	if (draw->GetDeviceList().size() > 1) {
+		std::string *deviceNameSetting = nullptr;
+		if (g_Config.iGPUBackend == (int)GPUBackend::VULKAN) {
+			deviceNameSetting = &g_Config.sVulkanDevice;
+		}
+#ifdef _WIN32
+		if (g_Config.iGPUBackend == (int)GPUBackend::DIRECT3D11) {
+			deviceNameSetting = &g_Config.sD3D11Device;
+		}
+#endif
+		if (deviceNameSetting) {
+			PopupMultiChoiceDynamic *deviceChoice = graphicsSettings->Add(new PopupMultiChoiceDynamic(deviceNameSetting, gr->T("Device"), draw->GetDeviceList(), nullptr, screenManager()));
+			deviceChoice->OnChoice.Handle(this, &GameSettingsScreen::OnRenderingBackend);
+		}
+	}
 
 	static const char *renderingMode[] = { "Non-Buffered Rendering", "Buffered Rendering"};
 	PopupMultiChoice *renderingModeChoice = graphicsSettings->Add(new PopupMultiChoice(&g_Config.iRenderingMode, gr->T("Mode"), renderingMode, 0, ARRAY_SIZE(renderingMode), gr->GetName(), screenManager()));

Windows/GPU/D3D11Context.cpp

@@ -2,6 +2,7 @@
 
 #include "Common/CommonWindows.h"
 #include <d3d11.h>
+#include <WinError.h>
 #include <cassert>
 
 #include "base/logging.h"
@@ -22,6 +23,12 @@
 #define __uuidof(type) IID_##type
 #endif
 
+#ifndef DXGI_ERROR_NOT_FOUND
+#define _FACDXGI    0x87a
+#define MAKE_DXGI_HRESULT(code) MAKE_HRESULT(1, _FACDXGI, code)
+#define DXGI_ERROR_NOT_FOUND MAKE_DXGI_HRESULT(2)
+#endif
+
 #if PPSSPP_PLATFORM(UWP)
 #error This file should not be compiled for UWP.
 #endif
@@ -41,7 +48,7 @@ void D3D11Context::SwapInterval(int interval) {
 	// Dummy
 }
 
-HRESULT D3D11Context::CreateTheDevice() {
+HRESULT D3D11Context::CreateTheDevice(IDXGIAdapter *adapter) {
 	bool windowed = true;
 #ifdef _DEBUG
 	UINT createDeviceFlags = D3D11_CREATE_DEVICE_DEBUG;
@@ -70,19 +77,13 @@ HRESULT D3D11Context::CreateTheDevice() {
 	const UINT numFeatureLevels = ARRAYSIZE(featureLevels);
 
 	HRESULT hr = S_OK;
-	// Temporarily commenting out until we can dynamically load D3D11CreateDevice.
-	for (UINT driverTypeIndex = 0; driverTypeIndex < numDriverTypes; driverTypeIndex++) {
-		driverType_ = driverTypes[driverTypeIndex];
-		hr = ptr_D3D11CreateDevice(nullptr, driverType_, nullptr, createDeviceFlags, (D3D_FEATURE_LEVEL *)featureLevels, numFeatureLevels,
+	D3D_DRIVER_TYPE driverType = D3D_DRIVER_TYPE_UNKNOWN;
+	hr = ptr_D3D11CreateDevice(adapter, driverType, nullptr, createDeviceFlags, (D3D_FEATURE_LEVEL *)featureLevels, numFeatureLevels,
+		D3D11_SDK_VERSION, &device_, &featureLevel_, &context_);
+	if (hr == E_INVALIDARG) {
+		// DirectX 11.0 platforms will not recognize D3D_FEATURE_LEVEL_11_1 so we need to retry without it
+		hr = ptr_D3D11CreateDevice(adapter, driverType, nullptr, createDeviceFlags, (D3D_FEATURE_LEVEL *)&featureLevels[3], numFeatureLevels - 3,
 			D3D11_SDK_VERSION, &device_, &featureLevel_, &context_);
-
-		if (hr == E_INVALIDARG) {
-			// DirectX 11.0 platforms will not recognize D3D_FEATURE_LEVEL_11_1 so we need to retry without it
-			hr = ptr_D3D11CreateDevice(nullptr, driverType_, nullptr, createDeviceFlags, (D3D_FEATURE_LEVEL *)&featureLevels[3], numFeatureLevels - 3,
-				D3D11_SDK_VERSION, &device_, &featureLevel_, &context_);
-		}
-		if (SUCCEEDED(hr))
-			break;
 	}
 	return hr;
 }
@@ -99,8 +100,30 @@ bool D3D11Context::Init(HINSTANCE hInst, HWND wnd, std::string *error_message) {
 	LoadD3D11Error result = LoadD3D11();
 
 	HRESULT hr = E_FAIL;
+	std::vector<std::string> adapterNames;
 	if (result == LoadD3D11Error::SUCCESS) {
-		hr = CreateTheDevice();
+		std::vector<IDXGIAdapter *> adapters;
+		int chosenAdapter = 0;
+
+		IDXGIFactory * pFactory = nullptr;
+		ptr_CreateDXGIFactory(__uuidof(IDXGIFactory), (void**)&pFactory);
+
+		IDXGIAdapter *pAdapter;
+		for (UINT i = 0; pFactory->EnumAdapters(i, &pAdapter) != DXGI_ERROR_NOT_FOUND; i++) {
+			adapters.push_back(pAdapter);
+			DXGI_ADAPTER_DESC desc;
+			pAdapter->GetDesc(&desc);
+			std::string str = ConvertWStringToUTF8(desc.Description);
+			adapterNames.push_back(str);
+			if (str == g_Config.sD3D11Device) {
+				chosenAdapter = i;
+			}
+		}
+
+		hr = CreateTheDevice(adapters[chosenAdapter]);
+		for (int i = 0; i < (int)adapters.size(); i++) {
+			adapters[i]->Release();
+		}
 	}
 
 	if (FAILED(hr)) {
@@ -146,7 +169,7 @@ bool D3D11Context::Init(HINSTANCE hInst, HWND wnd, std::string *error_message) {
 	}
 #endif
 
-	draw_ = Draw::T3DCreateD3D11Context(device_, context_, device1_, context1_, featureLevel_, hWnd_);
+	draw_ = Draw::T3DCreateD3D11Context(device_, context_, device1_, context1_, featureLevel_, hWnd_, adapterNames);
 	SetGPUBackend(GPUBackend::DIRECT3D11);
 	bool success = draw_->CreatePresets();  // If we can run D3D11, there's a compiler installed. I think.
 	assert(success);

Windows/GPU/D3D11Context.h

@@ -40,7 +40,7 @@ class D3D11Context : public WindowsGraphicsContext {
 	Draw::DrawContext *GetDrawContext() override { return draw_; }
 
 private:
-	HRESULT CreateTheDevice();
+	HRESULT CreateTheDevice(IDXGIAdapter *adapter);
 
 	void LostBackbuffer();
 	void GotBackbuffer();
@@ -60,12 +60,11 @@ class D3D11Context : public WindowsGraphicsContext {
 	ID3D11InfoQueue *d3dInfoQueue_ = nullptr;
 #endif
 
-	D3D_DRIVER_TYPE driverType_;
 	D3D_FEATURE_LEVEL featureLevel_ = D3D_FEATURE_LEVEL_11_0;
 	int adapterId;
 	HDC hDC;     // Private GDI Device Context
 	HWND hWnd_;   // Holds Our Window Handle
 	HMODULE hD3D11;
 	int width;
 	int height;
-};
+};

Windows/GPU/WindowsVulkanContext.cpp

@@ -109,7 +109,12 @@ bool WindowsVulkanContext::Init(HINSTANCE hInst, HWND hWnd, std::string *error_m
 		g_Vulkan = nullptr;
 		return false;
 	}
-	g_Vulkan->ChooseDevice(g_Vulkan->GetBestPhysicalDevice());
+	int deviceNum = g_Vulkan->GetPhysicalDeviceByName(g_Config.sVulkanDevice);
+	if (deviceNum < 0) {
+		deviceNum = g_Vulkan->GetBestPhysicalDevice();
+		g_Config.sVulkanDevice = g_Vulkan->GetPhysicalDeviceProperties(deviceNum).deviceName;
+	}
+	g_Vulkan->ChooseDevice(deviceNum);
 	if (g_Vulkan->EnableDeviceExtension(VK_NV_DEDICATED_ALLOCATION_EXTENSION_NAME)) {
 		supportsDedicatedAlloc_ = true;
 	}

ext/native/thin3d/VulkanQueueRunner.cpp

@@ -850,7 +850,7 @@ void VulkanQueueRunner::PerformBindFramebufferAsRenderTarget(const VKRStep &step
 		// See pull request #10723.
 		bool maliBugWorkaround = step.render.numDraws == 0 &&
 			step.render.color == VKRRenderPassAction::CLEAR &&
-			vulkan_->GetPhysicalDeviceProperties().driverVersion == 0xaa9c4b29;
+			vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).driverVersion == 0xaa9c4b29;
 		if (maliBugWorkaround) {
 			TransitionImageLayout2(cmd, step.render.framebuffer->color.image, 0, 1, VK_IMAGE_ASPECT_COLOR_BIT,
 				fb->color.layout, VK_IMAGE_LAYOUT_GENERAL,

ext/native/thin3d/VulkanRenderManager.cpp

@@ -136,7 +136,7 @@ VulkanRenderManager::VulkanRenderManager(VulkanContext *vulkan) : vulkan_(vulkan
 	queueRunner_.CreateDeviceObjects();
 
 	// Temporary AMD hack for issue #10097
-	if (vulkan_->GetPhysicalDeviceProperties().vendorID == VULKAN_VENDOR_AMD) {
+	if (vulkan_->GetPhysicalDeviceProperties(vulkan_->GetCurrentPhysicalDevice()).vendorID == VULKAN_VENDOR_AMD) {
 		useThread_ = false;
 	}
 }

ext/native/thin3d/thin3d.h

@@ -491,6 +491,7 @@ struct DeviceCaps {
 	bool framebufferBlitSupported;
 	bool framebufferDepthCopySupported;
 	bool framebufferDepthBlitSupported;
+	std::string deviceName;  // The device name to use when creating the thin3d context, to get the same one.
 };
 
 struct TextureDesc {
@@ -532,6 +533,7 @@ class DrawContext {
 	virtual uint32_t GetDataFormatSupport(DataFormat fmt) const = 0;
 	virtual std::vector<std::string> GetFeatureList() const { return std::vector<std::string>(); }
 	virtual std::vector<std::string> GetExtensionList() const { return std::vector<std::string>(); }
+	virtual std::vector<std::string> GetDeviceList() const { return std::vector<std::string>(); }
 
 	virtual uint32_t GetSupportedShaderLanguages() const = 0;