Skip to content

Commit

Permalink
[RISCV] Select ISD::AVGCEILS/AVGFLOORS as vaadd. (#92839)
Browse files Browse the repository at this point in the history 
I think the behaviors are the same if this describes their behavior.

AVGFLOORS sign extends the inputs by 1 bit, adds them, then does an
arithmetic shift right by 1 before truncating to the original bit width.
This is vaadd with rdn rounding mode.

AVGCEILS sign extends the inputs by 1 bit, adds them, then does an
arithmetic shift right by 1. If the bit shifted out is 1, it adds 1 to
the shifted value. Then truncates to the original bit width. This is vaadd
with rnu rounding mode.

I think this wasn't implemented previously because there was some
confusion about what average means. Some may expect average to round
towards zero, but there is no way to do that in RISC-V or with the
SelectionDAG nodes. Related issue
riscv/riscv-v-spec#935
  • Loading branch information
@topperc
topperc authored May 21, 2024
1 parent 50dbbe5 commit 6246b49
Show file tree
Hide file tree
Showing 6 changed files with 42 additions and 30 deletions.
20 changes: 14 additions & 6 deletions llvm/lib/Target/RISCV/RISCVISelLowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -844,8 +844,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
VT, Custom);
setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
Custom);
setOperationAction({ISD::AVGFLOORU, ISD::AVGCEILU, ISD::SADDSAT,
ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT},
setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
ISD::SSUBSAT, ISD::USUBSAT},
VT, Legal);

// Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
Expand Down Expand Up @@ -1237,8 +1238,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom);

setOperationAction({ISD::AVGFLOORU, ISD::AVGCEILU, ISD::SADDSAT,
ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT},
setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
ISD::SSUBSAT, ISD::USUBSAT},
VT, Custom);

setOperationAction(ISD::VSELECT, VT, Custom);
Expand Down Expand Up @@ -5841,7 +5843,9 @@ static unsigned getRISCVVLOp(SDValue Op) {
OP_CASE(UADDSAT)
OP_CASE(SSUBSAT)
OP_CASE(USUBSAT)
OP_CASE(AVGFLOORS)
OP_CASE(AVGFLOORU)
OP_CASE(AVGCEILS)
OP_CASE(AVGCEILU)
OP_CASE(FADD)
OP_CASE(FSUB)
Expand Down Expand Up @@ -5956,7 +5960,7 @@ static bool hasMergeOp(unsigned Opcode) {
Opcode <= RISCVISD::LAST_RISCV_STRICTFP_OPCODE &&
"not a RISC-V target specific op");
static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP ==
126 &&
128 &&
RISCVISD::LAST_RISCV_STRICTFP_OPCODE -
ISD::FIRST_TARGET_STRICTFP_OPCODE ==
21 &&
Expand All @@ -5982,7 +5986,7 @@ static bool hasMaskOp(unsigned Opcode) {
Opcode <= RISCVISD::LAST_RISCV_STRICTFP_OPCODE &&
"not a RISC-V target specific op");
static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP ==
126 &&
128 &&
RISCVISD::LAST_RISCV_STRICTFP_OPCODE -
ISD::FIRST_TARGET_STRICTFP_OPCODE ==
21 &&
Expand Down Expand Up @@ -6882,7 +6886,9 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
!Subtarget.hasVInstructionsF16()))
return SplitVectorOp(Op, DAG);
[[fallthrough]];
case ISD::AVGFLOORS:
case ISD::AVGFLOORU:
case ISD::AVGCEILS:
case ISD::AVGCEILU:
case ISD::SMIN:
case ISD::SMAX:
Expand Down Expand Up @@ -19958,7 +19964,9 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
NODE_NAME_CASE(UDIV_VL)
NODE_NAME_CASE(UREM_VL)
NODE_NAME_CASE(XOR_VL)
NODE_NAME_CASE(AVGFLOORS_VL)
NODE_NAME_CASE(AVGFLOORU_VL)
NODE_NAME_CASE(AVGCEILS_VL)
NODE_NAME_CASE(AVGCEILU_VL)
NODE_NAME_CASE(SADDSAT_VL)
NODE_NAME_CASE(UADDSAT_VL)
Expand Down
4 changes: 4 additions & 0 deletions llvm/lib/Target/RISCV/RISCVISelLowering.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,8 +264,12 @@ enum NodeType : unsigned {
SSUBSAT_VL,
USUBSAT_VL,

// Averaging adds of signed integers.
AVGFLOORS_VL,
// Averaging adds of unsigned integers.
AVGFLOORU_VL,
// Rounding averaging adds of signed integers.
AVGCEILS_VL,
// Rounding averaging adds of unsigned integers.
AVGCEILU_VL,

Expand Down
12 changes: 7 additions & 5 deletions llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -881,17 +881,17 @@ multiclass VPatMultiplyAddSDNode_VV_VX<SDNode op, string instruction_name> {
}
}

multiclass VPatAVGADD_VV_VX_RM<SDNode vop, int vxrm> {
multiclass VPatAVGADD_VV_VX_RM<SDNode vop, int vxrm, string suffix = ""> {
foreach vti = AllIntegerVectors in {
let Predicates = GetVTypePredicates<vti>.Predicates in {
def : Pat<(vop (vti.Vector vti.RegClass:$rs1),
(vti.Vector vti.RegClass:$rs2)),
(!cast<Instruction>("PseudoVAADDU_VV_"#vti.LMul.MX)
(!cast<Instruction>("PseudoVAADD"#suffix#"_VV_"#vti.LMul.MX)
(vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1, vti.RegClass:$rs2,
vxrm, vti.AVL, vti.Log2SEW, TA_MA)>;
def : Pat<(vop (vti.Vector vti.RegClass:$rs1),
(vti.Vector (SplatPat (XLenVT GPR:$rs2)))),
(!cast<Instruction>("PseudoVAADDU_VX_"#vti.LMul.MX)
(!cast<Instruction>("PseudoVAADD"#suffix#"_VX_"#vti.LMul.MX)
(vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1, GPR:$rs2,
vxrm, vti.AVL, vti.Log2SEW, TA_MA)>;
}
Expand Down Expand Up @@ -1163,8 +1163,10 @@ defm : VPatBinarySDNode_VV_VX<ssubsat, "PseudoVSSUB">;
defm : VPatBinarySDNode_VV_VX<usubsat, "PseudoVSSUBU">;

// 12.2. Vector Single-Width Averaging Add and Subtract
defm : VPatAVGADD_VV_VX_RM<avgflooru, 0b10>;
defm : VPatAVGADD_VV_VX_RM<avgceilu, 0b00>;
defm : VPatAVGADD_VV_VX_RM<avgfloors, 0b10>;
defm : VPatAVGADD_VV_VX_RM<avgflooru, 0b10, suffix = "U">;
defm : VPatAVGADD_VV_VX_RM<avgceils, 0b00>;
defm : VPatAVGADD_VV_VX_RM<avgceilu, 0b00, suffix = "U">;

// 12.5. Vector Narrowing Fixed-Point Clip Instructions
multiclass VPatTruncSatClipSDNode<VTypeInfo vti, VTypeInfo wti> {
Expand Down
14 changes: 9 additions & 5 deletions llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,9 @@ def riscv_ctlz_vl : SDNode<"RISCVISD::CTLZ_VL", SDT_RISCVIntUnOp_VL>
def riscv_cttz_vl : SDNode<"RISCVISD::CTTZ_VL", SDT_RISCVIntUnOp_VL>;
def riscv_ctpop_vl : SDNode<"RISCVISD::CTPOP_VL", SDT_RISCVIntUnOp_VL>;

def riscv_avgfloors_vl : SDNode<"RISCVISD::AVGFLOORS_VL", SDT_RISCVIntBinOp_VL, [SDNPCommutative]>;
def riscv_avgflooru_vl : SDNode<"RISCVISD::AVGFLOORU_VL", SDT_RISCVIntBinOp_VL, [SDNPCommutative]>;
def riscv_avgceils_vl : SDNode<"RISCVISD::AVGCEILS_VL", SDT_RISCVIntBinOp_VL, [SDNPCommutative]>;
def riscv_avgceilu_vl : SDNode<"RISCVISD::AVGCEILU_VL", SDT_RISCVIntBinOp_VL, [SDNPCommutative]>;
def riscv_saddsat_vl : SDNode<"RISCVISD::SADDSAT_VL", SDT_RISCVIntBinOp_VL, [SDNPCommutative]>;
def riscv_uaddsat_vl : SDNode<"RISCVISD::UADDSAT_VL", SDT_RISCVIntBinOp_VL, [SDNPCommutative]>;
Expand Down Expand Up @@ -2073,19 +2075,19 @@ multiclass VPatSlide1VL_VF<SDNode vop, string instruction_name> {
}
}

multiclass VPatAVGADDVL_VV_VX_RM<SDNode vop, int vxrm> {
multiclass VPatAVGADDVL_VV_VX_RM<SDNode vop, int vxrm, string suffix = ""> {
foreach vti = AllIntegerVectors in {
let Predicates = GetVTypePredicates<vti>.Predicates in {
def : Pat<(vop (vti.Vector vti.RegClass:$rs1),
(vti.Vector vti.RegClass:$rs2),
vti.RegClass:$merge, (vti.Mask V0), VLOpFrag),
(!cast<Instruction>("PseudoVAADDU_VV_"#vti.LMul.MX#"_MASK")
(!cast<Instruction>("PseudoVAADD"#suffix#"_VV_"#vti.LMul.MX#"_MASK")
vti.RegClass:$merge, vti.RegClass:$rs1, vti.RegClass:$rs2,
(vti.Mask V0), vxrm, GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
def : Pat<(vop (vti.Vector vti.RegClass:$rs1),
(vti.Vector (SplatPat (XLenVT GPR:$rs2))),
vti.RegClass:$merge, (vti.Mask V0), VLOpFrag),
(!cast<Instruction>("PseudoVAADDU_VX_"#vti.LMul.MX#"_MASK")
(!cast<Instruction>("PseudoVAADD"#suffix#"_VX_"#vti.LMul.MX#"_MASK")
vti.RegClass:$merge, vti.RegClass:$rs1, GPR:$rs2,
(vti.Mask V0), vxrm, GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
}
Expand Down Expand Up @@ -2369,8 +2371,10 @@ defm : VPatBinaryVL_VV_VX<riscv_ssubsat_vl, "PseudoVSSUB">;
defm : VPatBinaryVL_VV_VX<riscv_usubsat_vl, "PseudoVSSUBU">;

// 12.2. Vector Single-Width Averaging Add and Subtract
defm : VPatAVGADDVL_VV_VX_RM<riscv_avgflooru_vl, 0b10>;
defm : VPatAVGADDVL_VV_VX_RM<riscv_avgceilu_vl, 0b00>;
defm : VPatAVGADDVL_VV_VX_RM<riscv_avgfloors_vl, 0b10>;
defm : VPatAVGADDVL_VV_VX_RM<riscv_avgflooru_vl, 0b10, suffix="U">;
defm : VPatAVGADDVL_VV_VX_RM<riscv_avgceils_vl, 0b00>;
defm : VPatAVGADDVL_VV_VX_RM<riscv_avgceilu_vl, 0b00, suffix="U">;

// 12.5. Vector Narrowing Fixed-Point Clip Instructions
multiclass VPatTruncSatClipVL<VTypeInfo vti, VTypeInfo wti> {
Expand Down
11 changes: 4 additions & 7 deletions llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vaaddu.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ define <8 x i8> @vaaddu_vx_v8i8_floor(<8 x i8> %x, i8 %y) {
define <8 x i8> @vaaddu_vv_v8i8_floor_sexti16(<8 x i8> %x, <8 x i8> %y) {
; CHECK-LABEL: vaaddu_vv_v8i8_floor_sexti16:
; CHECK: # %bb.0:
; CHECK-NEXT: csrwi vxrm, 2
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vwadd.vv v10, v8, v9
; CHECK-NEXT: vnsrl.wi v8, v10, 1
; CHECK-NEXT: vaadd.vv v8, v8, v9
; CHECK-NEXT: ret
%xzv = sext <8 x i8> %x to <8 x i16>
%yzv = sext <8 x i8> %y to <8 x i16>
Expand Down Expand Up @@ -248,12 +248,9 @@ define <8 x i8> @vaaddu_vx_v8i8_ceil(<8 x i8> %x, i8 %y) {
define <8 x i8> @vaaddu_vv_v8i8_ceil_sexti16(<8 x i8> %x, <8 x i8> %y) {
; CHECK-LABEL: vaaddu_vv_v8i8_ceil_sexti16:
; CHECK: # %bb.0:
; CHECK-NEXT: csrwi vxrm, 0
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vwadd.vv v10, v8, v9
; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma
; CHECK-NEXT: vadd.vi v8, v10, 1
; CHECK-NEXT: vsetvli zero, zero, e8, mf2, ta, ma
; CHECK-NEXT: vnsrl.wi v8, v8, 1
; CHECK-NEXT: vaadd.vv v8, v8, v9
; CHECK-NEXT: ret
%xzv = sext <8 x i8> %x to <8 x i16>
%yzv = sext <8 x i8> %y to <8 x i16>
Expand Down
11 changes: 4 additions & 7 deletions llvm/test/CodeGen/RISCV/rvv/vaaddu-sdnode.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ define <vscale x 8 x i8> @vaaddu_vx_nxv8i8_floor(<vscale x 8 x i8> %x, i8 %y) {
define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_floor_sexti16(<vscale x 8 x i8> %x, <vscale x 8 x i8> %y) {
; CHECK-LABEL: vaaddu_vv_nxv8i8_floor_sexti16:
; CHECK: # %bb.0:
; CHECK-NEXT: csrwi vxrm, 2
; CHECK-NEXT: vsetvli a0, zero, e8, m1, ta, ma
; CHECK-NEXT: vwadd.vv v10, v8, v9
; CHECK-NEXT: vnsrl.wi v8, v10, 1
; CHECK-NEXT: vaadd.vv v8, v8, v9
; CHECK-NEXT: ret
%xzv = sext <vscale x 8 x i8> %x to <vscale x 8 x i16>
%yzv = sext <vscale x 8 x i8> %y to <vscale x 8 x i16>
Expand Down Expand Up @@ -226,12 +226,9 @@ define <vscale x 8 x i8> @vaaddu_vx_nxv8i8_ceil(<vscale x 8 x i8> %x, i8 %y) {
define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_ceil_sexti16(<vscale x 8 x i8> %x, <vscale x 8 x i8> %y) {
; CHECK-LABEL: vaaddu_vv_nxv8i8_ceil_sexti16:
; CHECK: # %bb.0:
; CHECK-NEXT: csrwi vxrm, 0
; CHECK-NEXT: vsetvli a0, zero, e8, m1, ta, ma
; CHECK-NEXT: vwadd.vv v10, v8, v9
; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; CHECK-NEXT: vadd.vi v10, v10, 1
; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
; CHECK-NEXT: vnsrl.wi v8, v10, 1
; CHECK-NEXT: vaadd.vv v8, v8, v9
; CHECK-NEXT: ret
%xzv = sext <vscale x 8 x i8> %x to <vscale x 8 x i16>
%yzv = sext <vscale x 8 x i8> %y to <vscale x 8 x i16>
Expand Down

0 comments on commit 6246b49

Please sign in to comment.