Merge pull request #9122 from Pokechu22/pr-4601-test

Fix Super Mario Sunshine debug cubes (originally PR #4601 by stenzek)

Source/Core/VideoBackends/Software/SWVertexLoader.cpp

@@ -18,6 +18,7 @@
 #include "VideoBackends/Software/Tev.h"
 #include "VideoBackends/Software/TransformUnit.h"
 
+#include "VideoCommon/CPMemory.h"
 #include "VideoCommon/DataReader.h"
 #include "VideoCommon/IndexGenerator.h"
 #include "VideoCommon/OpcodeDecoding.h"
@@ -69,9 +70,6 @@ void SWVertexLoader::DrawCurrentBatch(u32 base_index, u32 num_indices, u32 base_
     const u16 index = m_cpu_index_buffer[i];
     memset(static_cast<void*>(&m_vertex), 0, sizeof(m_vertex));
 
-    // Super Mario Sunshine requires those to be zero for those debug boxes.
-    m_vertex.color = {};
-
     // parse the videocommon format to our own struct format (m_vertex)
     SetFormat(g_main_cp_state.last_id, primitiveType);
     ParseVertex(VertexLoaderManager::GetCurrentVertexFormat()->GetVertexDeclaration(), index);
@@ -184,6 +182,39 @@ static void ReadVertexAttribute(T* dst, DataReader src, const AttributeFormat& f
   }
 }
 
+static void ParseColorAttributes(InputVertexData* dst, DataReader& src,
+                                 const PortableVertexDeclaration& vdec)
+{
+  const auto set_default_color = [](u8* color, int i) {
+    // The default alpha channel seems to depend on the number of components in the vertex format.
+    const auto& g0 = g_main_cp_state.vtx_attr[g_main_cp_state.last_id].g0;
+    const u32 color_elements = i == 0 ? g0.Color0Elements : g0.Color1Elements;
+    color[0] = color_elements == 0 ? 255 : 0;
+    color[1] = 255;
+    color[2] = 255;
+    color[3] = 255;
+  };
+
+  if (vdec.colors[0].enable)
+  {
+    // Use color0 for channel 0, and color1 for channel 1 if both colors 0 and 1 are present.
+    ReadVertexAttribute<u8>(dst->color[0].data(), src, vdec.colors[0], 0, 4, true);
+    if (vdec.colors[1].enable)
+      ReadVertexAttribute<u8>(dst->color[1].data(), src, vdec.colors[1], 0, 4, true);
+    else
+      set_default_color(dst->color[1].data(), 1);
+  }
+  else
+  {
+    // If only one of the color attributes is enabled, it is directed to color 0.
+    if (vdec.colors[1].enable)
+      ReadVertexAttribute<u8>(dst->color[0].data(), src, vdec.colors[1], 0, 4, true);
+    else
+      set_default_color(dst->color[0].data(), 0);
+    set_default_color(dst->color[1].data(), 1);
+  }
+}
+
 void SWVertexLoader::ParseVertex(const PortableVertexDeclaration& vdec, int index)
 {
   DataReader src(m_cpu_vertex_buffer.data(),
@@ -197,10 +228,7 @@ void SWVertexLoader::ParseVertex(const PortableVertexDeclaration& vdec, int inde
     ReadVertexAttribute<float>(&m_vertex.normal[i][0], src, vdec.normals[i], 0, 3, false);
   }
 
-  for (std::size_t i = 0; i < m_vertex.color.size(); i++)
-  {
-    ReadVertexAttribute<u8>(m_vertex.color[i].data(), src, vdec.colors[i], 0, 4, true);
-  }
+  ParseColorAttributes(&m_vertex, src, vdec);
 
   for (std::size_t i = 0; i < m_vertex.texCoords.size(); i++)
   {

Source/Core/VideoCommon/ConstantManager.h

@@ -58,7 +58,7 @@ struct VertexShaderConstants
   u32 components;           // .x
   u32 xfmem_dualTexInfo;    // .y
   u32 xfmem_numColorChans;  // .z
-  u32 pad1;                 // .w
+  u32 color_chan_alpha;     // .w
 
   std::array<float4, 6> posnormalmatrix;
   std::array<float4, 4> projection;

Source/Core/VideoCommon/GXPipelineTypes.h

@@ -20,7 +20,7 @@ namespace VideoCommon
 // As pipelines encompass both shader UIDs and render states, changes to either of these should
 // also increment the pipeline UID version. Incrementing the UID version will cause all UID
 // caches to be invalidated.
-constexpr u32 GX_PIPELINE_UID_VERSION = 1;  // Last changed in PR 6431
+constexpr u32 GX_PIPELINE_UID_VERSION = 2;  // Last changed in PR 9122
 
 struct GXPipelineUid
 {

Source/Core/VideoCommon/LightingShaderGen.cpp

@@ -78,7 +78,7 @@ static void GenerateLightShader(ShaderCode& object, const LightingUidData& uid_d
 // materials name is I_MATERIALS in vs and I_PMATERIALS in ps
 // inColorName is color in vs and colors_ in ps
 // dest is o.colors_ in vs and colors_ in ps
-void GenerateLightingShaderCode(ShaderCode& object, const LightingUidData& uid_data, int components,
+void GenerateLightingShaderCode(ShaderCode& object, const LightingUidData& uid_data,
                                 std::string_view in_color_name, std::string_view dest)
 {
   for (u32 j = 0; j < NUM_XF_COLOR_CHANNELS; j++)
@@ -87,43 +87,16 @@ void GenerateLightingShaderCode(ShaderCode& object, const LightingUidData& uid_d
 
     const bool colormatsource = !!(uid_data.matsource & (1 << j));
     if (colormatsource)  // from vertex
-    {
-      if ((components & (VB_HAS_COL0 << j)) != 0)
-        object.Write("int4 mat = int4(round({}{} * 255.0));\n", in_color_name, j);
-      else if ((components & VB_HAS_COL0) != 0)
-        object.Write("int4 mat = int4(round({}0 * 255.0));\n", in_color_name);
-      else
-        object.Write("int4 mat = int4(255, 255, 255, 255);\n");
-    }
+      object.Write("int4 mat = int4(round({}{} * 255.0));\n", in_color_name, j);
     else  // from color
-    {
       object.Write("int4 mat = {}[{}];\n", I_MATERIALS, j + 2);
-    }
 
     if ((uid_data.enablelighting & (1 << j)) != 0)
     {
       if ((uid_data.ambsource & (1 << j)) != 0)  // from vertex
-      {
-        if ((components & (VB_HAS_COL0 << j)) != 0)
-        {
-          object.Write("lacc = int4(round({}{} * 255.0));\n", in_color_name, j);
-        }
-        else if ((components & VB_HAS_COL0) != 0)
-        {
-          object.Write("lacc = int4(round({}0 * 255.0));\n", in_color_name);
-        }
-        else
-        {
-          // TODO: this isn't verified. Here we want to read the ambient from the vertex,
-          // but the vertex itself has no color. So we don't know which value to read.
-          // Returning 1.0 is the same as disabled lightning, so this could be fine
-          object.Write("lacc = int4(255, 255, 255, 255);\n");
-        }
-      }
+        object.Write("lacc = int4(round({}{} * 255.0));\n", in_color_name, j);
       else  // from color
-      {
         object.Write("lacc = {}[{}];\n", I_MATERIALS, j);
-      }
     }
     else
     {
@@ -135,42 +108,17 @@ void GenerateLightingShaderCode(ShaderCode& object, const LightingUidData& uid_d
     if (alphamatsource != colormatsource)
     {
       if (alphamatsource)  // from vertex
-      {
-        if ((components & (VB_HAS_COL0 << j)) != 0)
-          object.Write("mat.w = int(round({}{}.w * 255.0));\n", in_color_name, j);
-        else if ((components & VB_HAS_COL0) != 0)
-          object.Write("mat.w = int(round({}0.w * 255.0));\n", in_color_name);
-        else
-          object.Write("mat.w = 255;\n");
-      }
+        object.Write("mat.w = int(round({}{}.w * 255.0));\n", in_color_name, j);
       else  // from color
-      {
         object.Write("mat.w = {}[{}].w;\n", I_MATERIALS, j + 2);
-      }
     }
 
     if ((uid_data.enablelighting & (1 << (j + 2))) != 0)
     {
       if ((uid_data.ambsource & (1 << (j + 2))) != 0)  // from vertex
-      {
-        if ((components & (VB_HAS_COL0 << j)) != 0)
-        {
-          object.Write("lacc.w = int(round({}{}.w * 255.0));\n", in_color_name, j);
-        }
-        else if ((components & VB_HAS_COL0) != 0)
-        {
-          object.Write("lacc.w = int(round({}0.w * 255.0));\n", in_color_name);
-        }
-        else
-        {
-          // TODO: The same for alpha: We want to read from vertex, but the vertex has no color
-          object.Write("lacc.w = 255;\n");
-        }
-      }
+        object.Write("lacc.w = int(round({}{}.w * 255.0));\n", in_color_name, j);
       else  // from color
-      {
         object.Write("lacc.w = {}[{}].w;\n", I_MATERIALS, j);
-      }
     }
     else
     {

Source/Core/VideoCommon/LightingShaderGen.h

@@ -45,6 +45,6 @@ constexpr inline char s_lighting_struct[] = "struct Light {\n"
                                             "\tfloat4 dir;\n"
                                             "};\n";
 
-void GenerateLightingShaderCode(ShaderCode& object, const LightingUidData& uid_data, int components,
+void GenerateLightingShaderCode(ShaderCode& object, const LightingUidData& uid_data,
                                 std::string_view in_color_name, std::string_view dest);
 void GetLightingShaderUid(LightingUidData& uid_data);

Source/Core/VideoCommon/PixelShaderGen.cpp

@@ -203,9 +203,6 @@ PixelShaderUid GetPixelShaderUid()
 
   if (g_ActiveConfig.bEnablePixelLighting)
   {
-    // The lighting shader only needs the two color bits of the 23bit component bit array.
-    uid_data->components =
-        (VertexLoaderManager::g_current_components & (VB_HAS_COL0 | VB_HAS_COL1)) >> VB_COL_SHIFT;
     uid_data->numColorChans = xfmem.numChan.numColorChans;
     GetLightingShaderUid(uid_data->lighting);
   }
@@ -768,8 +765,11 @@ ShaderCode GeneratePixelShaderCode(APIType api_type, const ShaderHostConfig& hos
     // out.SetConstantsUsed(C_PLIGHT_COLORS, C_PLIGHT_COLORS+7); // TODO: Can be optimized further
     // out.SetConstantsUsed(C_PLIGHTS, C_PLIGHTS+31); // TODO: Can be optimized further
     // out.SetConstantsUsed(C_PMATERIALS, C_PMATERIALS+3);
-    GenerateLightingShaderCode(out, uid_data->lighting, uid_data->components << VB_COL_SHIFT,
-                               "colors_", "col");
+    GenerateLightingShaderCode(out, uid_data->lighting, "colors_", "col");
+    if (uid_data->numColorChans == 0)
+      out.Write("col0 = float4(0.0, 0.0, 0.0, 0.0);\n");
+    if (uid_data->numColorChans <= 1)
+      out.Write("col1 = float4(0.0, 0.0, 0.0, 0.0);\n");
   }
 
   // HACK to handle cases where the tex gen is not enabled

Source/Core/VideoCommon/PixelShaderGen.h

@@ -17,8 +17,7 @@ struct pixel_shader_uid_data
 
   u32 num_values;  // TODO: Shouldn't be a u32
   u32 NumValues() const { return num_values; }
-  u32 components : 2;
-  u32 pad0 : 2;
+  u32 pad0 : 4;
   u32 useDstAlpha : 1;
   u32 Pretest : 2;
   u32 nIndirectStagesUsed : 4;

Source/Core/VideoCommon/ShaderGenCommon.h

@@ -229,6 +229,7 @@ const char* GetInterpolationQualifier(bool msaa, bool ssaa, bool in_glsl_interfa
 static const char s_shader_uniforms[] = "\tuint    components;\n"
                                         "\tuint    xfmem_dualTexInfo;\n"
                                         "\tuint    xfmem_numColorChans;\n"
+                                        "\tuint    color_chan_alpha;\n"
                                         "\tfloat4 " I_POSNORMALMATRIX "[6];\n"
                                         "\tfloat4 " I_PROJECTION "[4];\n"
                                         "\tint4 " I_MATERIALS "[4];\n"

Source/Core/VideoCommon/UberShaderCommon.cpp

@@ -103,42 +103,23 @@ void WriteVertexLighting(ShaderCode& out, APIType api_type, std::string_view wor
             "  int4 lacc = int4(255, 255, 255, 255);\n"
             "\n");
 
-  out.Write("  if ({} != 0u) {{\n", BitfieldExtract("colorreg", LitChannel().matsource));
-  out.Write("    if ((components & ({}u << chan)) != 0u) // VB_HAS_COL0\n", VB_HAS_COL0);
-  out.Write("      mat.xyz = int3(round(((chan == 0u) ? {}.xyz : {}.xyz) * 255.0));\n",
+  out.Write("  if ({} != 0u)\n", BitfieldExtract("colorreg", LitChannel().matsource));
+  out.Write("    mat.xyz = int3(round(((chan == 0u) ? {}.xyz : {}.xyz) * 255.0));\n",
             in_color_0_var, in_color_1_var);
-  out.Write("    else if ((components & {}u) != 0u) // VB_HAS_COLO0\n", VB_HAS_COL0);
-  out.Write("      mat.xyz = int3(round({}.xyz * 255.0));\n", in_color_0_var);
-  out.Write("    else\n"
-            "      mat.xyz = int3(255, 255, 255);\n"
-            "  }}\n"
-            "\n");
 
-  out.Write("  if ({} != 0u) {{\n", BitfieldExtract("alphareg", LitChannel().matsource));
-  out.Write("    if ((components & ({}u << chan)) != 0u) // VB_HAS_COL0\n", VB_HAS_COL0);
-  out.Write("      mat.w = int(round(((chan == 0u) ? {}.w : {}.w) * 255.0));\n", in_color_0_var,
+  out.Write("  if ({} != 0u)\n", BitfieldExtract("alphareg", LitChannel().matsource));
+  out.Write("    mat.w = int(round(((chan == 0u) ? {}.w : {}.w) * 255.0));\n", in_color_0_var,
             in_color_1_var);
-  out.Write("    else if ((components & {}u) != 0u) // VB_HAS_COLO0\n", VB_HAS_COL0);
-  out.Write("      mat.w = int(round({}.w * 255.0));\n", in_color_0_var);
-  out.Write("    else\n"
-            "      mat.w = 255;\n"
-            "  }} else {{\n"
+  out.Write("  else\n"
             "    mat.w = " I_MATERIALS " [chan + 2u].w;\n"
-            "  }}\n"
             "\n");
 
   out.Write("  if ({} != 0u) {{\n", BitfieldExtract("colorreg", LitChannel().enablelighting));
-  out.Write("    if ({} != 0u) {{\n", BitfieldExtract("colorreg", LitChannel().ambsource));
-  out.Write("      if ((components & ({}u << chan)) != 0u) // VB_HAS_COL0\n", VB_HAS_COL0);
-  out.Write("        lacc.xyz = int3(round(((chan == 0u) ? {}.xyz : {}.xyz) * 255.0));\n",
+  out.Write("    if ({} != 0u)\n", BitfieldExtract("colorreg", LitChannel().ambsource));
+  out.Write("      lacc.xyz = int3(round(((chan == 0u) ? {}.xyz : {}.xyz) * 255.0));\n",
             in_color_0_var, in_color_1_var);
-  out.Write("      else if ((components & {}u) != 0u) // VB_HAS_COLO0\n", VB_HAS_COL0);
-  out.Write("        lacc.xyz = int3(round({}.xyz * 255.0));\n", in_color_0_var);
-  out.Write("      else\n"
-            "        lacc.xyz = int3(255, 255, 255);\n"
-            "    }} else {{\n"
+  out.Write("    else\n"
             "      lacc.xyz = " I_MATERIALS " [chan].xyz;\n"
-            "    }}\n"
             "\n");
   out.Write("    uint light_mask = {} | ({} << 4u);\n",
             BitfieldExtract("colorreg", LitChannel().lightMask0_3),

Source/Core/VideoCommon/UberShaderVertex.cpp

@@ -176,8 +176,43 @@ ShaderCode GenVertexShader(APIType api_type, const ShaderHostConfig& host_config
             "\n");
 
   // Hardware Lighting
-  WriteVertexLighting(out, api_type, "pos.xyz", "_norm0", "rawcolor0", "rawcolor1", "o.colors_0",
-                      "o.colors_1");
+  out.Write("// xfmem.numColorChans controls the number of color channels available to TEV,\n"
+            "// but we still need to generate all channels here, as it can be used in texgen.\n"
+            "// Cel-damage is an example of this.\n"
+            "float4 vertex_color_0, vertex_color_1;\n"
+            "\n");
+  out.Write("// To use color 1, the vertex descriptor must have color 0 and 1.\n"
+            "// If color 1 is present but not color 0, it is used for lighting channel 0.\n"
+            "bool use_color_1 = ((components & {0}) == {0}); // VB_HAS_COL0 | VB_HAS_COL1\n",
+            VB_HAS_COL0 | VB_HAS_COL1);
+
+  out.Write("for (uint color = 0; color < {}; color++) {{\n", NUM_XF_COLOR_CHANNELS);
+  out.Write("  if ((color == 0 || use_color_1) && (components & ({} << color)) != 0) {{\n",
+            VB_HAS_COL0);
+  out.Write("    float4 color_value;\n"
+            "    // Use color0 for channel 0, and color1 for channel 1 if both colors 0 and 1 are "
+            "present.\n"
+            "    if (color == 0u)\n"
+            "      vertex_color_0 = rawcolor0;\n"
+            "    else\n"
+            "      vertex_color_1 = rawcolor1;\n"
+            "  }} else if (color == 0 && (components & {}) != 0) {{\n",
+            VB_HAS_COL1);
+  out.Write("    // Use color1 for channel 0 if color0 is not present.\n"
+            "    vertex_color_0 = rawcolor1;\n"
+            "  }} else {{\n"
+            "    // The default alpha channel depends on the number of components in the vertex.\n"
+            "    float alpha = float((color_chan_alpha >> color) & 1u);\n"
+            "    if (color == 0u)\n"
+            "      vertex_color_0 = float4(1.0, 1.0, 1.0, alpha);\n"
+            "    else\n"
+            "      vertex_color_1 = float4(1.0, 1.0, 1.0, alpha);\n"
+            "  }}\n"
+            "}}\n"
+            "\n");
+
+  WriteVertexLighting(out, api_type, "pos.xyz", "_norm0", "vertex_color_0", "vertex_color_1",
+                      "o.colors_0", "o.colors_1");
 
   // Texture Coordinates
   if (num_texgen > 0)
@@ -207,14 +242,27 @@ ShaderCode GenVertexShader(APIType api_type, const ShaderHostConfig& host_config
   if (per_pixel_lighting)
   {
     out.Write("o.Normal = _norm0;\n"
-              "o.WorldPos = pos.xyz;\n");
+              "o.WorldPos = pos.xyz;\n"
+              "// Pass through the vertex colors unmodified so we can evaluate the lighting\n"
+              "// in the same manner.\n");
     out.Write("if ((components & {}u) != 0u) // VB_HAS_COL0\n"
-              "  o.colors_0 = rawcolor0;\n",
+              "  o.colors_0 = vertex_color_0;\n",
               VB_HAS_COL0);
     out.Write("if ((components & {}u) != 0u) // VB_HAS_COL1\n"
-              "  o.colors_1 = rawcolor1;\n",
+              "  o.colors_1 = vertex_color_1;\n",
               VB_HAS_COL1);
   }
+  else
+  {
+    out.Write("// The number of colors available to TEV is determined by numColorChans.\n"
+              "// We have to provide the fields to match the interface, so set to zero\n"
+              "// if it's not enabled.\n"
+              "if (xfmem_numColorChans == 0u)\n"
+              "  o.colors_0 = float4(0.0, 0.0, 0.0, 0.0);\n"
+              "if (xfmem_numColorChans <= 1u)\n"
+              "  o.colors_1 = float4(0.0, 0.0, 0.0, 0.0);\n"
+              "\n");
+  }
 
   // If we can disable the incorrect depth clipping planes using depth clamping, then we can do
   // our own depth clipping and calculate the depth range before the perspective divide if