rsx: Simplify support for ABGR formats

rpcs3/Emu/RSX/Common/TextureUtils.cpp

@@ -1022,7 +1022,7 @@ std::pair<u32, bool> get_compatible_gcm_format(rsx::surface_color_format format)
 	case rsx::surface_color_format::x8b8g8r8_o8b8g8r8:
 	case rsx::surface_color_format::x8b8g8r8_z8b8g8r8:
 	case rsx::surface_color_format::a8b8g8r8:
-		return{ CELL_GCM_TEXTURE_A8R8G8B8, false };
+		return{ CELL_GCM_TEXTURE_A8R8G8B8, true };
 
 	case rsx::surface_color_format::w16z16y16x16:
 		return{ CELL_GCM_TEXTURE_W16_Z16_Y16_X16_FLOAT, true };

rpcs3/Emu/RSX/GL/GLPresent.cpp

@@ -1,5 +1,6 @@
 #include "stdafx.h"
 #include "GLGSRender.h"
+#include "Emu/Cell/Modules/cellVideoOut.h"
 
 LOG_CHANNEL(screenshot);
 
@@ -76,7 +77,21 @@ gl::texture* GLGSRender::get_present_source(gl::present_surface_info* info, cons
 		const auto range = utils::address_range::start_length(info->address, info->pitch * info->height);
 		m_gl_texture_cache.invalidate_range(cmd, range, rsx::invalidation_cause::read);
 
-		m_flip_tex_color->copy_from(vm::base(info->address), gl::texture::format::bgra, gl::texture::type::uint_8_8_8_8, unpack_settings);
+		gl::texture::format fmt;
+		switch (avconfig->format)
+		{
+		default:
+			rsx_log.error("Unhandled video output format 0x%x", avconfig->format);
+			[[fallthrough]];
+		case CELL_VIDEO_OUT_BUFFER_COLOR_FORMAT_X8R8G8B8:
+			fmt = gl::texture::format::bgra;
+			break;
+		case CELL_VIDEO_OUT_BUFFER_COLOR_FORMAT_X8B8G8R8:
+			fmt = gl::texture::format::rgba;
+			break;
+		}
+
+		m_flip_tex_color->copy_from(vm::base(info->address), fmt, gl::texture::type::uint_8_8_8_8, unpack_settings);
 		image = m_flip_tex_color.get();
 	}
 
@@ -205,14 +220,14 @@ void GLGSRender::flip(const rsx::display_flip_info_t& info)
 			pack_settings.apply();
 
 			if (gl::get_driver_caps().ARB_dsa_supported)
-				glGetTextureImage(image_to_flip, 0, GL_BGRA, GL_UNSIGNED_BYTE, buffer_height * buffer_width * 4, sshot_frame.data());
+				glGetTextureImage(image_to_flip, 0, GL_RGBA, GL_UNSIGNED_BYTE, buffer_height * buffer_width * 4, sshot_frame.data());
 			else
-				glGetTextureImageEXT(image_to_flip, GL_TEXTURE_2D, 0, GL_BGRA, GL_UNSIGNED_BYTE, sshot_frame.data());
+				glGetTextureImageEXT(image_to_flip, GL_TEXTURE_2D, 0, GL_RGBA, GL_UNSIGNED_BYTE, sshot_frame.data());
 
 			if (GLenum err; (err = glGetError()) != GL_NO_ERROR)
 				screenshot.error("Failed to capture image: 0x%x", err);
 			else
-				m_frame->take_screenshot(std::move(sshot_frame), buffer_width, buffer_height);
+				m_frame->take_screenshot(std::move(sshot_frame), buffer_width, buffer_height, false);
 		}
 
 		const areai screen_area = coordi({}, { static_cast<int>(buffer_width), static_cast<int>(buffer_height) });

rpcs3/Emu/RSX/GL/GLTexture.cpp

@@ -1,6 +1,7 @@
 #include "stdafx.h"
 #include "GLTexture.h"
 #include "GLCompute.h"
+#include "GLRenderTargets.h"
 #include "../GCM.h"
 #include "../RSXThread.h"
 #include "../RSXTexture.h"
@@ -133,6 +134,29 @@ namespace gl
 		}
 	}
 
+	pixel_buffer_layout get_format_type(const gl::texture* tex)
+	{
+		const auto ifmt = tex->get_internal_format();
+		if (ifmt == gl::texture::internal_format::rgba8)
+		{
+			// Multiple RTT layouts can map to this format. Override ABGR formats
+			if (auto rtt = dynamic_cast<const gl::render_target*>(tex))
+			{
+				switch (rtt->format_info.gcm_color_format)
+				{
+				case rsx::surface_color_format::x8b8g8r8_z8b8g8r8:
+				case rsx::surface_color_format::x8b8g8r8_o8b8g8r8:
+				case rsx::surface_color_format::a8b8g8r8:
+					return { GL_RGBA, GL_UNSIGNED_INT_8_8_8_8, 4, false };
+				default:
+					break;
+				}
+			}
+		}
+
+		return get_format_type(ifmt);
+	}
+
 	GLenum get_srgb_format(GLenum in_format)
 	{
 		switch (in_format)
@@ -780,8 +804,8 @@ namespace gl
 		}
 
 		const auto& caps = gl::get_driver_caps();
-		const auto pack_info = get_format_type(src->get_internal_format());
-		const auto unpack_info = get_format_type(dst->get_internal_format());
+		auto pack_info = get_format_type(src);
+		auto unpack_info = get_format_type(dst);
 
 		// Start pack operation
 		g_typeless_transfer_buffer.bind(buffer::target::pixel_pack);

rpcs3/Emu/RSX/GSRender.h

@@ -90,7 +90,7 @@ class GSFrameBase
 	virtual display_handle_t handle() const = 0;
 
 	std::atomic<bool> screenshot_toggle = false;
-	virtual void take_screenshot(const std::vector<u8> sshot_data, const u32 sshot_width, const u32 sshot_height) = 0;
+	virtual void take_screenshot(const std::vector<u8> sshot_data, const u32 sshot_width, const u32 sshot_height, bool is_bgra) = 0;
 };
 
 class GSRender : public rsx::thread

rpcs3/Emu/RSX/VK/VKFormats.cpp

@@ -28,6 +28,17 @@ namespace vk
 		vkGetPhysicalDeviceFormatProperties(dev, VK_FORMAT_B8G8R8A8_UNORM, &props);
 		result.bgra8_linear = !!(props.linearTilingFeatures & VK_FORMAT_FEATURE_BLIT_SRC_BIT);
 
+		// Check if device supports RGBA8 format
+		vkGetPhysicalDeviceFormatProperties(dev, VK_FORMAT_R8G8B8A8_UNORM, &props);
+		if (!(props.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT) ||
+			!(props.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT) ||
+			!(props.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_SRC_BIT))
+		{
+			// Non-fatal. Most games use BGRA layout due to legacy reasons as old GPUs typically supported BGRA and RGBA was emulated.
+			rsx_log.error("Your GPU and/or driver does not support RGBA8 format. This can cause problems in some rare games that use this memory layout.");
+		}
+
+		result.argb8_linear = !!(props.linearTilingFeatures & VK_FORMAT_FEATURE_BLIT_SRC_BIT);
 		return result;
 	}
 
@@ -395,10 +406,11 @@ namespace vk
 			//8-bit
 		case VK_FORMAT_R8_UNORM:
 		case VK_FORMAT_A8B8G8R8_UNORM_PACK32:
-		case VK_FORMAT_R8G8B8A8_UNORM:
 			return{ false, 1 };
 		case VK_FORMAT_B8G8R8A8_UNORM:
+		case VK_FORMAT_R8G8B8A8_UNORM:
 		case VK_FORMAT_B8G8R8A8_SRGB:
+		case VK_FORMAT_R8G8B8A8_SRGB:
 			return{ true, 4 };
 			//16-bit
 		case VK_FORMAT_R16_UINT:

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -42,11 +42,8 @@ namespace vk
 
 	std::pair<VkFormat, VkComponentMapping> get_compatible_surface_format(rsx::surface_color_format color_format)
 	{
-		const VkComponentMapping abgr = { VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_A };
 		const VkComponentMapping o_rgb = { VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_ONE };
 		const VkComponentMapping z_rgb = { VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_ZERO };
-		const VkComponentMapping o_bgr = { VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_ONE };
-		const VkComponentMapping z_bgr = { VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_ZERO };
 
 		switch (color_format)
 		{
@@ -57,13 +54,13 @@ namespace vk
 			return std::make_pair(VK_FORMAT_B8G8R8A8_UNORM, vk::default_component_map());
 
 		case rsx::surface_color_format::a8b8g8r8:
-			return std::make_pair(VK_FORMAT_B8G8R8A8_UNORM, abgr);
+			return std::make_pair(VK_FORMAT_R8G8B8A8_UNORM, vk::default_component_map());
 
 		case rsx::surface_color_format::x8b8g8r8_o8b8g8r8:
-			return std::make_pair(VK_FORMAT_B8G8R8A8_UNORM, o_bgr);
+			return std::make_pair(VK_FORMAT_R8G8B8A8_UNORM, o_rgb);
 
 		case rsx::surface_color_format::x8b8g8r8_z8b8g8r8:
-			return std::make_pair(VK_FORMAT_B8G8R8A8_UNORM, z_bgr);
+			return std::make_pair(VK_FORMAT_R8G8B8A8_UNORM, z_rgb);
 
 		case rsx::surface_color_format::x8r8g8b8_z8r8g8b8:
 			return std::make_pair(VK_FORMAT_B8G8R8A8_UNORM, z_rgb);

rpcs3/Emu/RSX/VK/VKHelpers.h

@@ -269,6 +269,7 @@ namespace vk
 		bool d24_unorm_s8;
 		bool d32_sfloat_s8;
 		bool bgra8_linear;
+		bool argb8_linear;
 	};
 
 	struct gpu_shader_types_support

rpcs3/Emu/RSX/VK/VKPresent.cpp

@@ -1,6 +1,6 @@
 #include "stdafx.h"
 #include "VKGSRender.h"
-
+#include "Emu/Cell/Modules/cellVideoOut.h"
 
 void VKGSRender::reinitialize_swapchain()
 {
@@ -347,8 +347,22 @@ vk::image* VKGSRender::get_present_source(vk::present_surface_info* info, const
 			flush_command_queue();
 		}
 
+		VkFormat format;
+		switch (avconfig->format)
+		{
+		default:
+			rsx_log.error("Unhandled video output format 0x%x", avconfig->format);
+			[[fallthrough]];
+		case CELL_VIDEO_OUT_BUFFER_COLOR_FORMAT_X8R8G8B8:
+			format = VK_FORMAT_B8G8R8A8_UNORM;
+			break;
+		case CELL_VIDEO_OUT_BUFFER_COLOR_FORMAT_X8B8G8R8:
+			format = VK_FORMAT_R8G8B8A8_UNORM;
+			break;
+		}
+
 		m_texture_cache.invalidate_range(*m_current_command_buffer, range, rsx::invalidation_cause::read);
-		image_to_flip = m_texture_cache.upload_image_simple(*m_current_command_buffer, info->address, info->width, info->height, info->pitch);
+		image_to_flip = m_texture_cache.upload_image_simple(*m_current_command_buffer, format, info->address, info->width, info->height, info->pitch);
 	}
 
 	return image_to_flip;
@@ -645,7 +659,8 @@ void VKGSRender::flip(const rsx::display_flip_info_t& info)
 			memcpy(sshot_frame.data(), src, sshot_size);
 			sshot_vkbuf.unmap();
 
-			m_frame->take_screenshot(std::move(sshot_frame), buffer_width, buffer_height);
+			const bool is_bgra = image_to_flip->format() == VK_FORMAT_B8G8R8A8_UNORM;
+			m_frame->take_screenshot(std::move(sshot_frame), buffer_width, buffer_height, is_bgra);
 		}
 	}
 

rpcs3/Emu/RSX/VK/VKResolveHelper.cpp

@@ -12,6 +12,7 @@ namespace
 		{
 			case VK_FORMAT_R5G6B5_UNORM_PACK16:
 				return "r16ui";
+			case VK_FORMAT_R8G8B8A8_UNORM:
 			case VK_FORMAT_B8G8R8A8_UNORM:
 				return "rgba8";
 			case VK_FORMAT_R16G16B16A16_SFLOAT:

rpcs3/Emu/RSX/VK/VKTexture.cpp

@@ -7,6 +7,7 @@
 #include "VKFormats.h"
 #include "VKCompute.h"
 #include "VKRenderPass.h"
+#include "VKRenderTargets.h"
 
 namespace vk
 {
@@ -265,8 +266,8 @@ namespace vk
 		{
 			vk::copy_image_to_buffer(cmd, src, scratch_buf, src_copy);
 
-			const auto src_convert = get_format_convert_flags(src->info.format);
-			const auto dst_convert = get_format_convert_flags(dst->info.format);
+			auto src_convert = get_format_convert_flags(src->info.format);
+			auto dst_convert = get_format_convert_flags(dst->info.format);
 
 			if (src_convert.first || dst_convert.first)
 			{

rpcs3/Emu/RSX/VK/VKTextureCache.h

@@ -1471,9 +1471,24 @@ namespace vk
 			baseclass::on_frame_end();
 		}
 
-		vk::image *upload_image_simple(vk::command_buffer& cmd, u32 address, u32 width, u32 height, u32 pitch)
+		vk::image *upload_image_simple(vk::command_buffer& cmd, VkFormat format, u32 address, u32 width, u32 height, u32 pitch)
 		{
-			if (!m_formats_support.bgra8_linear)
+			bool linear_format_supported = false;
+
+			switch (format)
+			{
+			case VK_FORMAT_B8G8R8A8_UNORM:
+				linear_format_supported = m_formats_support.bgra8_linear;
+				break;
+			case VK_FORMAT_R8G8B8A8_UNORM:
+				linear_format_supported = m_formats_support.argb8_linear;
+				break;
+			default:
+				rsx_log.error("Unsupported VkFormat 0x%x" HERE, static_cast<u32>(format));
+				return nullptr;
+			}
+
+			if (!linear_format_supported)
 			{
 				return nullptr;
 			}
@@ -1482,7 +1497,7 @@ namespace vk
 			auto image = std::make_unique<vk::viewable_image>(*m_device, m_memory_types.host_visible_coherent,
 				VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT,
 				VK_IMAGE_TYPE_2D,
-				VK_FORMAT_B8G8R8A8_UNORM,
+				format,
 				width, height, 1, 1, 1, VK_SAMPLE_COUNT_1_BIT, VK_IMAGE_LAYOUT_PREINITIALIZED,
 				VK_IMAGE_TILING_LINEAR, VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_SAMPLED_BIT, 0);
 

rpcs3/rpcs3qt/gs_frame.cpp

@@ -335,10 +335,10 @@ void gs_frame::flip(draw_context_t, bool /*skip_frame*/)
 	}
 }
 
-void gs_frame::take_screenshot(const std::vector<u8> sshot_data, const u32 sshot_width, const u32 sshot_height)
+void gs_frame::take_screenshot(const std::vector<u8> sshot_data, const u32 sshot_width, const u32 sshot_height, bool is_bgra)
 {
 	std::thread(
-		[sshot_width, sshot_height](const std::vector<u8> sshot_data)
+		[sshot_width, sshot_height, is_bgra](const std::vector<u8> sshot_data)
 		{
 			std::string screen_path = fs::get_config_dir() + "screenshots/";
 
@@ -361,9 +361,19 @@ void gs_frame::take_screenshot(const std::vector<u8> sshot_data, const u32 sshot
 			const u32* sshot_ptr = reinterpret_cast<const u32*>(sshot_data.data());
 			u32* alpha_ptr       = reinterpret_cast<u32*>(sshot_data_alpha.data());
 
-			for (size_t index = 0; index < sshot_data.size() / sizeof(u32); index++)
+			if (is_bgra) [[likely]]
 			{
-				alpha_ptr[index] = ((sshot_ptr[index] & 0xFF) << 16) | (sshot_ptr[index] & 0xFF00) | ((sshot_ptr[index] & 0xFF0000) >> 16) | 0xFF000000;
+				for (size_t index = 0; index < sshot_data.size() / sizeof(u32); index++)
+				{
+					alpha_ptr[index] = ((sshot_ptr[index] & 0xFF) << 16) | (sshot_ptr[index] & 0xFF00) | ((sshot_ptr[index] & 0xFF0000) >> 16) | 0xFF000000;
+				}
+			}
+			else
+			{
+				for (size_t index = 0; index < sshot_data.size() / sizeof(u32); index++)
+				{
+					alpha_ptr[index] = sshot_ptr[index] | 0xFF000000;
+				}
 			}
 
 			std::vector<u8> encoded_png;

rpcs3/rpcs3qt/gs_frame.h

@@ -50,7 +50,7 @@ class gs_frame : public QWindow, public GSFrameBase
 	void progress_increment(int delta);
 	void progress_set_limit(int limit);
 
-	void take_screenshot(const std::vector<u8> sshot_data, const u32 sshot_width, const u32 sshot_height) override;
+	void take_screenshot(const std::vector<u8> sshot_data, const u32 sshot_width, const u32 sshot_height, bool is_bgra) override;
 
 protected:
 	virtual void paintEvent(QPaintEvent *event);