Make helper invocations take part in subgroupQuad operations

include/vkgcDefs.h

@@ -453,7 +453,6 @@ struct ShaderModuleUsage {
   bool enableVarPtrStorageBuf; ///< Whether to enable "VariablePointerStorageBuffer" capability
   bool enableVarPtr;           ///< Whether to enable "VariablePointer" capability
   bool useSubgroupSize;        ///< Whether gl_SubgroupSize is used
-  bool useHelpInvocation;      ///< Whether fragment shader has helper-invocation for subgroup
   bool useSpecConstant;        ///< Whether specialization constant is used
   bool keepUnusedFunctions;    ///< Whether to keep unused function
   bool useIsNan;               ///< Whether IsNan is used

lgc/builder/BuilderImpl.h

@@ -727,13 +727,13 @@ class SubgroupBuilder : virtual public BuilderImplBase {
   llvm::Value *CreateSubgroupElect(const llvm::Twine &instName) override final;
 
   // Create a subgroup all.
-  llvm::Value *CreateSubgroupAll(llvm::Value *const value, bool wqm, const llvm::Twine &instName) override final;
+  llvm::Value *CreateSubgroupAll(llvm::Value *const value, const llvm::Twine &instName) override final;
 
   // Create a subgroup any
-  llvm::Value *CreateSubgroupAny(llvm::Value *const value, bool wqm, const llvm::Twine &instName) override final;
+  llvm::Value *CreateSubgroupAny(llvm::Value *const value, const llvm::Twine &instName) override final;
 
   // Create a subgroup all equal.
-  llvm::Value *CreateSubgroupAllEqual(llvm::Value *const value, bool wqm, const llvm::Twine &instName) override final;
+  llvm::Value *CreateSubgroupAllEqual(llvm::Value *const value, const llvm::Twine &instName) override final;
 
   // Create a subgroup broadcast.
   llvm::Value *CreateSubgroupBroadcast(llvm::Value *const value, llvm::Value *const index,
@@ -854,6 +854,7 @@ class SubgroupBuilder : virtual public BuilderImplBase {
 
   llvm::Value *createDsSwizzle(llvm::Value *const value, uint16_t dsPattern);
   llvm::Value *createWwm(llvm::Value *const value);
+  llvm::Value *createWqm(llvm::Value *const value);
   llvm::Value *createThreadMask();
   llvm::Value *createThreadMaskedSelect(llvm::Value *const threadMask, uint64_t andMask, llvm::Value *const value1,
                                         llvm::Value *const value2);

lgc/builder/BuilderRecorder.cpp

@@ -1642,30 +1642,27 @@ Value *BuilderRecorder::CreateSubgroupElect(const Twine &instName) {
 // Create a subgroup all.
 //
 // @param value : The value to compare
-// @param wqm : Executed in WQM (whole quad mode)
 // @param instName : Name to give instruction(s)
-Value *BuilderRecorder::CreateSubgroupAll(Value *const value, bool wqm, const Twine &instName) {
-  return record(Opcode::SubgroupAll, getInt1Ty(), {value, getInt1(wqm)}, instName);
+Value *BuilderRecorder::CreateSubgroupAll(Value *const value, const Twine &instName) {
+  return record(Opcode::SubgroupAll, getInt1Ty(), {value}, instName);
 }
 
 // =====================================================================================================================
 // Create a subgroup any
 //
 // @param value : The value to compare
-// @param wqm : Executed in WQM (whole quad mode)
 // @param instName : Name to give instruction(s)
-Value *BuilderRecorder::CreateSubgroupAny(Value *const value, bool wqm, const Twine &instName) {
-  return record(Opcode::SubgroupAny, getInt1Ty(), {value, getInt1(wqm)}, instName);
+Value *BuilderRecorder::CreateSubgroupAny(Value *const value, const Twine &instName) {
+  return record(Opcode::SubgroupAny, getInt1Ty(), {value}, instName);
 }
 
 // =====================================================================================================================
 // Create a subgroup all equal.
 //
 // @param value : The value to compare
-// @param wqm : Executed in WQM (whole quad mode)
 // @param instName : Name to give instruction(s)
-Value *BuilderRecorder::CreateSubgroupAllEqual(Value *const value, bool wqm, const Twine &instName) {
-  return record(Opcode::SubgroupAllEqual, getInt1Ty(), {value, getInt1(wqm)}, instName);
+Value *BuilderRecorder::CreateSubgroupAllEqual(Value *const value, const Twine &instName) {
+  return record(Opcode::SubgroupAllEqual, getInt1Ty(), {value}, instName);
 }
 
 // =====================================================================================================================

lgc/builder/BuilderReplayer.cpp

@@ -790,13 +790,13 @@ Value *BuilderReplayer::processCall(unsigned opcode, CallInst *call) {
     return m_builder->CreateSubgroupElect();
   }
   case BuilderRecorder::Opcode::SubgroupAll: {
-    return m_builder->CreateSubgroupAll(args[0], cast<ConstantInt>(args[1])->getZExtValue() != 0);
+    return m_builder->CreateSubgroupAll(args[0]);
   }
   case BuilderRecorder::Opcode::SubgroupAny: {
-    return m_builder->CreateSubgroupAny(args[0], cast<ConstantInt>(args[1])->getZExtValue() != 0);
+    return m_builder->CreateSubgroupAny(args[0]);
   }
   case BuilderRecorder::Opcode::SubgroupAllEqual: {
-    return m_builder->CreateSubgroupAllEqual(args[0], cast<ConstantInt>(args[1])->getZExtValue() != 0);
+    return m_builder->CreateSubgroupAllEqual(args[0]);
   }
   case BuilderRecorder::Opcode::SubgroupBroadcast: {
     return m_builder->CreateSubgroupBroadcast(args[0], args[1]);

lgc/builder/SubgroupBuilder.cpp

@@ -83,14 +83,13 @@ Value *SubgroupBuilder::CreateSubgroupElect(const Twine &instName) {
 // Create a subgroup all call.
 //
 // @param value : The value to compare across the subgroup. Must be an integer type.
-// @param wqm : Executed in WQM (whole quad mode)
 // @param instName : Name to give final instruction.
-Value *SubgroupBuilder::CreateSubgroupAll(Value *const value, bool wqm, const Twine &instName) {
+Value *SubgroupBuilder::CreateSubgroupAll(Value *const value, const Twine &instName) {
   Value *result = CreateICmpEQ(createGroupBallot(value), createGroupBallot(getTrue()));
   result = CreateSelect(CreateUnaryIntrinsic(Intrinsic::is_constant, value), value, result);
 
   // Helper invocations of whole quad mode should be included in the subgroup vote execution
-  if (wqm) {
+  if (m_shaderStage == ShaderStageFragment) {
     result = CreateZExt(result, getInt32Ty());
     result = CreateIntrinsic(Intrinsic::amdgcn_softwqm, {getInt32Ty()}, {result});
     result = CreateTrunc(result, getInt1Ty());
@@ -102,14 +101,13 @@ Value *SubgroupBuilder::CreateSubgroupAll(Value *const value, bool wqm, const Tw
 // Create a subgroup any call.
 //
 // @param value : The value to compare across the subgroup. Must be an integer type.
-// @param wqm : Executed in WQM (whole quad mode)
 // @param instName : Name to give final instruction.
-Value *SubgroupBuilder::CreateSubgroupAny(Value *const value, bool wqm, const Twine &instName) {
+Value *SubgroupBuilder::CreateSubgroupAny(Value *const value, const Twine &instName) {
   Value *result = CreateICmpNE(createGroupBallot(value), getInt64(0));
   result = CreateSelect(CreateUnaryIntrinsic(Intrinsic::is_constant, value), value, result);
 
   // Helper invocations of whole quad mode should be included in the subgroup vote execution
-  if (wqm) {
+  if (m_shaderStage == ShaderStageFragment) {
     result = CreateZExt(result, getInt32Ty());
     result = CreateIntrinsic(Intrinsic::amdgcn_softwqm, {getInt32Ty()}, {result});
     result = CreateTrunc(result, getInt1Ty());
@@ -121,9 +119,8 @@ Value *SubgroupBuilder::CreateSubgroupAny(Value *const value, bool wqm, const Tw
 // Create a subgroup all equal call.
 //
 // @param value : The value to compare across the subgroup. Must be an integer type.
-// @param wqm : Executed in WQM (whole quad mode)
 // @param instName : Name to give final instruction.
-Value *SubgroupBuilder::CreateSubgroupAllEqual(Value *const value, bool wqm, const Twine &instName) {
+Value *SubgroupBuilder::CreateSubgroupAllEqual(Value *const value, const Twine &instName) {
   Type *const type = value->getType();
 
   Value *compare = CreateSubgroupBroadcastFirst(value, instName);
@@ -141,9 +138,9 @@ Value *SubgroupBuilder::CreateSubgroupAllEqual(Value *const value, bool wqm, con
     for (unsigned i = 1, compCount = cast<FixedVectorType>(type)->getNumElements(); i < compCount; i++)
       result = CreateAnd(result, CreateExtractElement(compare, i));
 
-    return CreateSubgroupAll(result, wqm, instName);
+    return CreateSubgroupAll(result, instName);
   } else
-    return CreateSubgroupAll(compare, wqm, instName);
+    return CreateSubgroupAll(compare, instName);
 }
 
 // =====================================================================================================================
@@ -1004,7 +1001,7 @@ Value *SubgroupBuilder::CreateSubgroupQuadBroadcast(Value *const value, Value *c
     result = CreateSelect(compare, createDsSwizzle(value, getDsSwizzleQuadMode(3, 3, 3, 3)), result);
   }
 
-  return result;
+  return createWqm(result);
 }
 
 // =====================================================================================================================
@@ -1014,9 +1011,9 @@ Value *SubgroupBuilder::CreateSubgroupQuadBroadcast(Value *const value, Value *c
 // @param instName : Name to give final instruction.
 Value *SubgroupBuilder::CreateSubgroupQuadSwapHorizontal(Value *const value, const Twine &instName) {
   if (supportDpp())
-    return createDppMov(value, DppCtrl::DppQuadPerm1032, 0xF, 0xF, true);
-  else
-    return createDsSwizzle(value, getDsSwizzleQuadMode(1, 0, 3, 2));
+    return createWqm(createDppMov(value, DppCtrl::DppQuadPerm1032, 0xF, 0xF, true));
+
+  return createWqm(createDsSwizzle(value, getDsSwizzleQuadMode(1, 0, 3, 2)));
 }
 
 // =====================================================================================================================
@@ -1026,9 +1023,9 @@ Value *SubgroupBuilder::CreateSubgroupQuadSwapHorizontal(Value *const value, con
 // @param instName : Name to give final instruction.
 Value *SubgroupBuilder::CreateSubgroupQuadSwapVertical(Value *const value, const Twine &instName) {
   if (supportDpp())
-    return createDppMov(value, DppCtrl::DppQuadPerm2301, 0xF, 0xF, true);
-  else
-    return createDsSwizzle(value, getDsSwizzleQuadMode(2, 3, 0, 1));
+    return createWqm(createDppMov(value, DppCtrl::DppQuadPerm2301, 0xF, 0xF, true));
+
+  return createWqm(createDsSwizzle(value, getDsSwizzleQuadMode(2, 3, 0, 1)));
 }
 
 // =====================================================================================================================
@@ -1038,9 +1035,9 @@ Value *SubgroupBuilder::CreateSubgroupQuadSwapVertical(Value *const value, const
 // @param instName : Name to give final instruction.
 Value *SubgroupBuilder::CreateSubgroupQuadSwapDiagonal(Value *const value, const Twine &instName) {
   if (supportDpp())
-    return createDppMov(value, DppCtrl::DppQuadPerm3210, 0xF, 0xF, true);
-  else
-    return createDsSwizzle(value, getDsSwizzleQuadMode(3, 2, 1, 0));
+    return createWqm(createDppMov(value, DppCtrl::DppQuadPerm3210, 0xF, 0xF, true));
+
+  return createWqm(createDsSwizzle(value, getDsSwizzleQuadMode(3, 2, 1, 0)));
 }
 
 // =====================================================================================================================
@@ -1358,6 +1355,22 @@ Value *SubgroupBuilder::createWwm(Value *const value) {
   return CreateMapToInt32(mapFunc, value, {});
 }
 
+// =====================================================================================================================
+// Create a call to WQM (whole quad mode).
+// Only in fragment shader stage.
+//
+// @param value : The value to pass to the soft WQM call.
+Value *SubgroupBuilder::createWqm(Value *const value) {
+  auto mapFunc = [](BuilderBase &builder, ArrayRef<Value *> mappedArgs, ArrayRef<Value *>) -> Value * {
+    return builder.CreateUnaryIntrinsic(Intrinsic::amdgcn_wqm, mappedArgs[0]);
+  };
+
+  if (m_shaderStage == ShaderStageFragment)
+    return CreateMapToInt32(mapFunc, value, {});
+
+  return value;
+}
+
 // =====================================================================================================================
 // Create a ds_swizzle bit mode pattern.
 //

lgc/include/lgc/builder/BuilderRecorder.h

@@ -544,9 +544,9 @@ class BuilderRecorder final : public Builder, BuilderRecorderMetadataKinds {
   llvm::Value *CreateGetWaveSize(const llvm::Twine &instName) override final;
   llvm::Value *CreateGetSubgroupSize(const llvm::Twine &instName) override final;
   llvm::Value *CreateSubgroupElect(const llvm::Twine &instName) override final;
-  llvm::Value *CreateSubgroupAll(llvm::Value *const value, bool wqm, const llvm::Twine &instName) override final;
-  llvm::Value *CreateSubgroupAny(llvm::Value *const value, bool wqm, const llvm::Twine &instName) override final;
-  llvm::Value *CreateSubgroupAllEqual(llvm::Value *const value, bool wqm, const llvm::Twine &instName) override final;
+  llvm::Value *CreateSubgroupAll(llvm::Value *const value, const llvm::Twine &instName) override final;
+  llvm::Value *CreateSubgroupAny(llvm::Value *const value, const llvm::Twine &instName) override final;
+  llvm::Value *CreateSubgroupAllEqual(llvm::Value *const value, const llvm::Twine &instName) override final;
   llvm::Value *CreateSubgroupBroadcast(llvm::Value *const value, llvm::Value *const index,
                                        const llvm::Twine &instName) override final;
   llvm::Value *CreateSubgroupBroadcastWaterfall(llvm::Value *const value, llvm::Value *const index,

lgc/interface/lgc/Builder.h

@@ -1455,26 +1455,20 @@ class Builder : public BuilderCommon {
   // Create a subgroup all.
   //
   // @param value : The value to compare
-  // @param wqm : Executed in WQM (whole quad mode)
   // @param instName : Name to give instruction(s)
-  virtual llvm::Value *CreateSubgroupAll(llvm::Value *const value, bool wqm = false,
-                                         const llvm::Twine &instName = "") = 0;
+  virtual llvm::Value *CreateSubgroupAll(llvm::Value *const value, const llvm::Twine &instName = "") = 0;
 
   // Create a subgroup any
   //
   // @param value : The value to compare
-  // @param wqm : Executed in WQM (whole quad mode)
   // @param instName : Name to give instruction(s)
-  virtual llvm::Value *CreateSubgroupAny(llvm::Value *const value, bool wqm = false,
-                                         const llvm::Twine &instName = "") = 0;
+  virtual llvm::Value *CreateSubgroupAny(llvm::Value *const value, const llvm::Twine &instName = "") = 0;
 
   // Create a subgroup all equal.
   //
   // @param value : The value to compare
-  // @param wqm : Executed in WQM (whole quad mode)
   // @param instName : Name to give instruction(s)
-  virtual llvm::Value *CreateSubgroupAllEqual(llvm::Value *const value, bool wqm = false,
-                                              const llvm::Twine &instName = "") = 0;
+  virtual llvm::Value *CreateSubgroupAllEqual(llvm::Value *const value, const llvm::Twine &instName = "") = 0;
 
   // Create a subgroup broadcast.
   //

llpc/test/shaderdb/extensions/ExtSubgroupQuad_TestSubgroupQuadBroadcast.frag

@@ -0,0 +1,45 @@
+// BEGIN_SHADERTEST
+/*
+; RUN: amdllpc -v %gfxip %s | FileCheck -check-prefix=SHADERTEST %s
+; SHADERTEST-LABEL: {{^// LLPC.*}} SPIR-V lowering results
+; SHADERTEST: call reassoc nnan nsz arcp contract afn float (...) @lgc.create.subgroup.quad.broadcast.f32
+; SHADERTEST: call reassoc nnan nsz arcp contract afn float (...) @lgc.create.subgroup.quad.broadcast.f32
+; SHADERTEST: call reassoc nnan nsz arcp contract afn float (...) @lgc.create.subgroup.quad.broadcast.f32
+; SHADERTEST: call reassoc nnan nsz arcp contract afn float (...) @lgc.create.subgroup.quad.broadcast.f32
+; SHADERTEST-LABEL: {{^// LLPC}} pipeline patching results
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: AMDLLPC SUCCESS
+*/
+// END_SHADERTEST
+
+#version 450
+
+#extension GL_KHR_shader_subgroup_quad : require
+
+layout(binding = 0) readonly buffer Block0
+{
+    float alpha[];
+};
+
+layout(location = 0) out vec4 color;
+
+void main()
+{
+    ivec2 coord = ivec2(gl_FragCoord.xy);
+    float v = alpha[coord.y * 2 + coord.x];
+
+    vec4 lanes;
+    lanes.x = subgroupQuadBroadcast(v, 0u);
+    lanes.y = subgroupQuadBroadcast(v, 1u);
+    lanes.z = subgroupQuadBroadcast(v, 2u);
+    lanes.w = subgroupQuadBroadcast(v, 3u);
+
+    color = lanes;
+}

llpc/test/shaderdb/extensions/ExtSubgroupQuad_TestSubgroupQuadSwapDiagonal.frag

@@ -0,0 +1,33 @@
+// BEGIN_SHADERTEST
+/*
+; RUN: amdllpc -v %gfxip %s | FileCheck -check-prefix=SHADERTEST %s
+; SHADERTEST-LABEL: {{^// LLPC.*}} SPIR-V lowering results
+; SHADERTEST: call reassoc nnan nsz arcp contract afn <4 x float> (...) @lgc.create.subgroup.quad.swap.diagonal.v4f32
+; SHADERTEST-LABEL: {{^// LLPC}} pipeline patching results
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: AMDLLPC SUCCESS
+*/
+// END_SHADERTEST
+
+#version 450
+
+#extension GL_KHR_shader_subgroup_quad : require
+
+layout(location = 0) out vec4 result;
+
+layout(set = 0, binding = 4, std430) readonly buffer Buffer0
+{
+  vec4 data[];
+};
+
+void main (void)
+{
+  result = subgroupQuadSwapDiagonal(data[gl_SubgroupInvocationID]);
+}

llpc/test/shaderdb/extensions/ExtSubgroupQuad_TestSubgroupQuadSwapVertical.frag

@@ -0,0 +1,33 @@
+// BEGIN_SHADERTEST
+/*
+; RUN: amdllpc -v %gfxip %s | FileCheck -check-prefix=SHADERTEST %s
+; SHADERTEST-LABEL: {{^// LLPC.*}} SPIR-V lowering results
+; SHADERTEST: call reassoc nnan nsz arcp contract afn <4 x float> (...) @lgc.create.subgroup.quad.swap.vertical.v4f32
+; SHADERTEST-LABEL: {{^// LLPC}} pipeline patching results
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.mov.dpp.i32
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: call i32 @llvm.amdgcn.wqm
+; SHADERTEST: AMDLLPC SUCCESS
+*/
+// END_SHADERTEST
+
+#version 450
+
+#extension GL_KHR_shader_subgroup_quad : require
+
+layout(location = 0) out vec4 result;
+
+layout(set = 0, binding = 4, std430) readonly buffer Buffer0
+{
+  vec4 data[];
+};
+
+void main (void)
+{
+  result = subgroupQuadSwapVertical(data[gl_SubgroupInvocationID]);
+}