Vulkan: Implement stencil upload (for Star Ocean).

CMakeLists.txt

@@ -1174,6 +1174,7 @@ set(GPU_VULKAN
 	GPU/Vulkan/ShaderManagerVulkan.h
 	GPU/Vulkan/StateMappingVulkan.cpp
 	GPU/Vulkan/StateMappingVulkan.h
+	GPU/Vulkan/StencilBufferVulkan.cpp
 	GPU/Vulkan/TextureCacheVulkan.cpp
 	GPU/Vulkan/TextureCacheVulkan.h
 	GPU/Vulkan/TextureScalerVulkan.cpp

GPU/GPU.vcxproj

@@ -366,6 +366,7 @@
     <ClCompile Include="Vulkan\PipelineManagerVulkan.cpp" />
     <ClCompile Include="Vulkan\ShaderManagerVulkan.cpp" />
     <ClCompile Include="Vulkan\StateMappingVulkan.cpp" />
+    <ClCompile Include="Vulkan\StencilBufferVulkan.cpp" />
     <ClCompile Include="Vulkan\TextureCacheVulkan.cpp" />
     <ClCompile Include="Vulkan\TextureScalerVulkan.cpp" />
     <ClCompile Include="Vulkan\VertexShaderGeneratorVulkan.cpp" />

GPU/GPU.vcxproj.filters

@@ -525,5 +525,8 @@
     <ClCompile Include="Debugger\Record.cpp">
       <Filter>Debugger</Filter>
     </ClCompile>
+    <ClCompile Include="Vulkan\StencilBufferVulkan.cpp">
+      <Filter>Vulkan</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

GPU/Vulkan/FramebufferVulkan.cpp

@@ -147,6 +147,10 @@ void FramebufferManagerVulkan::DestroyDeviceObjects() {
 		vulkan_->Delete().QueueDeleteShaderModule(fsBasicTex_);
 	if (vsBasicTex_ != VK_NULL_HANDLE)
 		vulkan_->Delete().QueueDeleteShaderModule(vsBasicTex_);
+	if (stencilFs_ != VK_NULL_HANDLE)
+		vulkan_->Delete().QueueDeleteShaderModule(stencilFs_);
+	if (stencilVs_ != VK_NULL_HANDLE)
+		vulkan_->Delete().QueueDeleteShaderModule(stencilVs_);
 
 	if (linearSampler_ != VK_NULL_HANDLE)
 		vulkan_->Delete().QueueDeleteSampler(linearSampler_);
@@ -369,12 +373,6 @@ void FramebufferManagerVulkan::RebindFramebuffer() {
 	gstate_c.Dirty(DIRTY_VIEWPORTSCISSOR_STATE);
 }
 
-bool FramebufferManagerVulkan::NotifyStencilUpload(u32 addr, int size, bool skipZero) {
-	// In Vulkan we should be able to simply copy the stencil data directly to a stencil buffer without
-	// messing about with bitplane textures and the like. Or actually, maybe not...
-	return false;
-}
-
 int FramebufferManagerVulkan::GetLineWidth() {
 	if (g_Config.iInternalResolution == 0) {
 		return std::max(1, (int)(renderWidth_ / 480));

GPU/Vulkan/FramebufferVulkan.h

@@ -134,6 +134,9 @@ class FramebufferManagerVulkan : public FramebufferManagerCommon {
 	VkShaderModule fsBasicTex_;
 	VkShaderModule vsBasicTex_;
 
+	VkShaderModule stencilVs_ = VK_NULL_HANDLE;
+	VkShaderModule stencilFs_ = VK_NULL_HANDLE;
+
 	VkPipeline cur2DPipeline_ = VK_NULL_HANDLE;
 
 	// Postprocessing

GPU/Vulkan/StencilBufferVulkan.cpp

@@ -0,0 +1,219 @@
+// Copyright (c) 2014- PPSSPP Project.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 2.0 or later versions.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License 2.0 for more details.
+
+// A copy of the GPL 2.0 should have been included with the program.
+// If not, see http://www.gnu.org/licenses/
+
+// Official git repository and contact information can be found at
+// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
+
+#include "base/logging.h"
+
+#include "ext/native/thin3d/thin3d.h"
+#include "ext/native/thin3d/VulkanRenderManager.h"
+#include "Core/Reporting.h"
+#include "GPU/Vulkan/FramebufferVulkan.h"
+#include "GPU/Vulkan/FragmentShaderGeneratorVulkan.h"
+#include "GPU/Vulkan/ShaderManagerVulkan.h"
+#include "GPU/Vulkan/TextureCacheVulkan.h"
+#include "GPU/Vulkan/VulkanUtil.h"
+
+#define STR_HELPER(x) #x
+#define STR(x) STR_HELPER(x)
+
+struct StencilValueUB {
+	uint32_t u_stencilValue[4];
+};
+
+static const char *stencil_fs = R"(#version 400
+#extension GL_ARB_separate_shader_objects : enable
+#extension GL_ARB_shading_language_420pack : enable
+layout (binding = 0) uniform sampler2D tex;
+layout(push_constant) uniform params {
+  int u_stencilValue;
+};
+layout (location = 0) in vec2 v_texcoord0;
+layout (location = 0) out vec4 fragColor0;
+
+void main() {
+  vec4 index = texture(tex, v_texcoord0);
+	int indexBits = int(floor(index.a * 255.99)) & 0xFF;
+	if ((indexBits & u_stencilValue) == 0)
+		discard;
+  fragColor0 = index.aaaa;
+}
+)";
+
+static const char stencil_vs[] = R"(#version 400
+#extension GL_ARB_separate_shader_objects : enable
+#extension GL_ARB_shading_language_420pack : enable
+layout (location = 0) out vec2 v_texcoord0;
+out gl_PerVertex { vec4 gl_Position; };
+void main() {
+	int id = gl_VertexIndex;
+  v_texcoord0.x = (id == 2) ? 2.0 : 0.0;
+  v_texcoord0.y = (id == 1) ? 2.0 : 0.0;
+  gl_Position = vec4(v_texcoord0 * vec2(2.0, 2.0) + vec2(-1.0, -1.0), 0.0, 1.0);
+}
+)";
+
+static u8 StencilBits5551(const u8 *ptr8, u32 numPixels) {
+	const u32 *ptr = (const u32 *)ptr8;
+
+	for (u32 i = 0; i < numPixels / 2; ++i) {
+		if (ptr[i] & 0x80008000) {
+			return 1;
+		}
+	}
+	return 0;
+}
+
+static u8 StencilBits4444(const u8 *ptr8, u32 numPixels) {
+	const u32 *ptr = (const u32 *)ptr8;
+	u32 bits = 0;
+
+	for (u32 i = 0; i < numPixels / 2; ++i) {
+		bits |= ptr[i];
+	}
+
+	return ((bits >> 12) & 0xF) | (bits >> 28);
+}
+
+static u8 StencilBits8888(const u8 *ptr8, u32 numPixels) {
+	const u32 *ptr = (const u32 *)ptr8;
+	u32 bits = 0;
+
+	for (u32 i = 0; i < numPixels; ++i) {
+		bits |= ptr[i];
+	}
+
+	return bits >> 24;
+}
+
+// In Vulkan we should be able to simply copy the stencil data directly to a stencil buffer without
+// messing about with bitplane textures and the like. Or actually, maybe not... Let's start with
+// the traditional approach.
+bool FramebufferManagerVulkan::NotifyStencilUpload(u32 addr, int size, bool skipZero) {
+	if (!MayIntersectFramebuffer(addr)) {
+		return false;
+	}
+
+	VirtualFramebuffer *dstBuffer = 0;
+	for (size_t i = 0; i < vfbs_.size(); ++i) {
+		VirtualFramebuffer *vfb = vfbs_[i];
+		if (MaskedEqual(vfb->fb_address, addr)) {
+			dstBuffer = vfb;
+		}
+	}
+	if (!dstBuffer) {
+		return false;
+	}
+
+	int values = 0;
+	u8 usedBits = 0;
+
+	const u8 *src = Memory::GetPointer(addr);
+	if (!src) {
+		return false;
+	}
+
+	switch (dstBuffer->format) {
+	case GE_FORMAT_565:
+		// Well, this doesn't make much sense.
+		return false;
+	case GE_FORMAT_5551:
+		usedBits = StencilBits5551(src, dstBuffer->fb_stride * dstBuffer->bufferHeight);
+		values = 2;
+		break;
+	case GE_FORMAT_4444:
+		usedBits = StencilBits4444(src, dstBuffer->fb_stride * dstBuffer->bufferHeight);
+		values = 16;
+		break;
+	case GE_FORMAT_8888:
+		usedBits = StencilBits8888(src, dstBuffer->fb_stride * dstBuffer->bufferHeight);
+		values = 256;
+		break;
+	case GE_FORMAT_INVALID:
+		// Impossible.
+		break;
+	}
+
+	std::string error;
+	if (!stencilVs_) {
+		stencilVs_ = CompileShaderModule(vulkan_, VK_SHADER_STAGE_VERTEX_BIT, stencil_vs, &error);
+		stencilFs_ = CompileShaderModule(vulkan_, VK_SHADER_STAGE_FRAGMENT_BIT, stencil_fs, &error);
+	}
+	VkRenderPass rp = (VkRenderPass)draw_->GetNativeObject(Draw::NativeObject::FRAMEBUFFER_RENDERPASS);
+
+	VulkanRenderManager *renderManager = (VulkanRenderManager *)draw_->GetNativeObject(Draw::NativeObject::RENDER_MANAGER);
+
+	if (usedBits == 0) {
+		if (skipZero) {
+			// Common when creating buffers, it's already 0.  We're done.
+			return false;
+		}
+
+		// TODO: Find a nice way to clear alpha here too.
+		draw_->BindFramebufferAsRenderTarget(dstBuffer->fbo, { Draw::RPAction::KEEP, Draw::RPAction::CLEAR });
+		gstate_c.Dirty(DIRTY_BLEND_STATE | DIRTY_RASTER_STATE | DIRTY_VIEWPORTSCISSOR_STATE);
+		return true;
+	}
+
+	shaderManagerVulkan_->DirtyLastShader();
+	textureCacheVulkan_->ForgetLastTexture();
+
+	u16 w = dstBuffer->renderWidth;
+	u16 h = dstBuffer->renderHeight;
+	float u1 = 1.0f;
+	float v1 = 1.0f;
+	MakePixelTexture(src, dstBuffer->format, dstBuffer->fb_stride, dstBuffer->bufferWidth, dstBuffer->bufferHeight, u1, v1);
+	if (dstBuffer->fbo) {
+		draw_->BindFramebufferAsRenderTarget(dstBuffer->fbo, { Draw::RPAction::KEEP, Draw::RPAction::CLEAR });
+	} else {
+		// something is wrong...
+	}
+
+	VkPipeline pipeline = vulkan2D_->GetPipeline(rp, stencilVs_, stencilFs_, false, Vulkan2D::VK2DDepthStencilMode::STENCIL_REPLACE_ALWAYS);
+	renderManager->BindPipeline(pipeline);
+	renderManager->SetViewport({ 0.0f, 0.0f, (float)w, (float)h, 0.0f, 1.0f });
+	renderManager->SetScissor({ { 0, 0, },{ (uint32_t)w, (uint32_t)h } });
+	gstate_c.Dirty(DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_BLEND_STATE | DIRTY_RASTER_STATE | DIRTY_DEPTHSTENCIL_STATE);
+
+	VkDescriptorSet descSet = vulkan2D_->GetDescriptorSet(overrideImageView_, nearestSampler_, VK_NULL_HANDLE, VK_NULL_HANDLE);
+
+	for (int i = 1; i < values; i += i) {
+		if (!(usedBits & i)) {
+			// It's already zero, let's skip it.
+			continue;
+		}
+		// These feel a little backwards : Mask is the bits that are going to be written, while value
+		// is the "mask" that will be tested against.
+		uint8_t mask = 0;
+		uint32_t value = 0;
+		if (dstBuffer->format == GE_FORMAT_4444) {
+			mask = i | (i << 4);
+			value = i * 16;
+		} else if (dstBuffer->format == GE_FORMAT_5551) {
+			mask = 0xFF;
+			value = i * 128;
+		} else {
+			mask = i;
+			value = i;
+		}
+		renderManager->SetStencilParams(mask, 0xFF, 0xFF);
+		renderManager->PushConstants(vulkan2D_->GetPipelineLayout(), VK_SHADER_STAGE_FRAGMENT_BIT, 0, 4, &value);
+		renderManager->Draw(vulkan2D_->GetPipelineLayout(), descSet, 0, nullptr, VK_NULL_HANDLE, 0, 3);  // full screen triangle
+	}
+
+	overrideImageView_ = VK_NULL_HANDLE;
+	RebindFramebuffer();
+	return true;
+}

GPU/Vulkan/VulkanUtil.cpp

@@ -83,12 +83,12 @@ void Vulkan2D::InitDeviceObjects() {
 	assert(VK_SUCCESS == res);
 
 	VkDescriptorPoolSize dpTypes[1];
-	dpTypes[0].descriptorCount = 1500;
+	dpTypes[0].descriptorCount = 3000;
 	dpTypes[0].type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
 
 	VkDescriptorPoolCreateInfo dp = { VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO };
 	dp.flags = 0;   // Don't want to mess around with individually freeing these, let's go fixed each frame and zap the whole array. Might try the dynamic approach later.
-	dp.maxSets = 1500;
+	dp.maxSets = 3000;
 	dp.pPoolSizes = dpTypes;
 	dp.poolSizeCount = ARRAY_SIZE(dpTypes);
 	for (int i = 0; i < ARRAY_SIZE(frameData_); i++) {
@@ -98,7 +98,7 @@ void Vulkan2D::InitDeviceObjects() {
 
 	VkPushConstantRange push = {};
 	push.offset = 0;
-	push.size = 32;
+	push.size = 16;
 	push.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT;
 
 	VkPipelineLayoutCreateInfo pl = { VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO };
@@ -195,11 +195,13 @@ VkDescriptorSet Vulkan2D::GetDescriptorSet(VkImageView tex1, VkSampler sampler1,
 	return desc;
 }
 
-VkPipeline Vulkan2D::GetPipeline(VkRenderPass rp, VkShaderModule vs, VkShaderModule fs) {
+VkPipeline Vulkan2D::GetPipeline(VkRenderPass rp, VkShaderModule vs, VkShaderModule fs, bool readVertices, VK2DDepthStencilMode depthStencilMode) {
 	PipelineKey key;
 	key.vs = vs;
 	key.fs = fs;
 	key.rp = rp;
+	key.depthStencilMode = depthStencilMode;
+	key.readVertices = readVertices;
 
 	auto iter = pipelines_.find(key);
 	if (iter != pipelines_.end()) {
@@ -208,7 +210,7 @@ VkPipeline Vulkan2D::GetPipeline(VkRenderPass rp, VkShaderModule vs, VkShaderMod
 
 	VkPipelineColorBlendAttachmentState blend0 = {};
 	blend0.blendEnable = false;
-	blend0.colorWriteMask = 0xF;
+	blend0.colorWriteMask = depthStencilMode == VK2DDepthStencilMode::STENCIL_REPLACE_ALWAYS ? 0 : 0xF;
 
 	VkPipelineColorBlendStateCreateInfo cbs = { VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO };
 	cbs.pAttachments = &blend0;
@@ -217,13 +219,32 @@ VkPipeline Vulkan2D::GetPipeline(VkRenderPass rp, VkShaderModule vs, VkShaderMod
 
 	VkPipelineDepthStencilStateCreateInfo dss = { VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO };
 	dss.depthBoundsTestEnable = false;
-	dss.stencilTestEnable = false;
 	dss.depthTestEnable = false;
+	dss.stencilTestEnable = false;
+	switch (depthStencilMode) {
+	case VK2DDepthStencilMode::NONE:
+		break;
+	case VK2DDepthStencilMode::STENCIL_REPLACE_ALWAYS:
+		dss.stencilTestEnable = true;
+		dss.front.reference = 0xFF;
+		dss.front.compareMask = 0xFF;
+		dss.front.compareOp = VK_COMPARE_OP_ALWAYS;
+		dss.front.depthFailOp = VK_STENCIL_OP_REPLACE;
+		dss.front.failOp = VK_STENCIL_OP_REPLACE;
+		dss.front.passOp = VK_STENCIL_OP_REPLACE;
+		dss.back = dss.front;
+		break;
+	}
 
-	VkDynamicState dynamicStates[2];
+	VkDynamicState dynamicStates[5];
 	int numDyn = 0;
 	dynamicStates[numDyn++] = VK_DYNAMIC_STATE_SCISSOR;
 	dynamicStates[numDyn++] = VK_DYNAMIC_STATE_VIEWPORT;
+	if (depthStencilMode == VK2DDepthStencilMode::STENCIL_REPLACE_ALWAYS) {
+		dynamicStates[numDyn++] = VK_DYNAMIC_STATE_STENCIL_WRITE_MASK;
+		dynamicStates[numDyn++] = VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK;
+		dynamicStates[numDyn++] = VK_DYNAMIC_STATE_STENCIL_REFERENCE;
+	}
 
 	VkPipelineDynamicStateCreateInfo ds = { VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO };
 	ds.pDynamicStates = dynamicStates;
@@ -270,10 +291,10 @@ VkPipeline Vulkan2D::GetPipeline(VkRenderPass rp, VkShaderModule vs, VkShaderMod
 	ibd.stride = vertexStride;
 
 	VkPipelineVertexInputStateCreateInfo vis = { VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO };
-	vis.vertexBindingDescriptionCount = 1;
-	vis.pVertexBindingDescriptions = &ibd;
-	vis.vertexAttributeDescriptionCount = attributeCount;
-	vis.pVertexAttributeDescriptions = attrs;
+	vis.vertexBindingDescriptionCount = readVertices ? 1 : 0;
+	vis.pVertexBindingDescriptions = readVertices ? &ibd : nullptr;
+	vis.vertexAttributeDescriptionCount = readVertices ? attributeCount : 0;
+	vis.pVertexAttributeDescriptions = readVertices ? attrs : nullptr;
 
 	VkPipelineViewportStateCreateInfo views = { VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO };
 	views.viewportCount = 1;
@@ -291,7 +312,6 @@ VkPipeline Vulkan2D::GetPipeline(VkRenderPass rp, VkShaderModule vs, VkShaderMod
 	pipe.pDepthStencilState = &dss;
 	pipe.pRasterizationState = &rs;
 
-	// We will use dynamic viewport state.
 	pipe.pVertexInputState = &vis;
 	pipe.pViewportState = &views;
 	pipe.pTessellationState = nullptr;