369 changes: 299 additions & 70 deletions llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
View file

Large diffs are not rendered by default.

106 changes: 99 additions & 7 deletions llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,23 @@ class VPatBinarySDNode_VV<SDPatternOperator vop,
op_reg_class:$rs2,
avl, sew)>;

class VPatBinarySDNode_VV_E<SDPatternOperator vop,
string instruction_name,
ValueType result_type,
ValueType op_type,
int log2sew,
LMULInfo vlmul,
int sew,
OutPatFrag avl,
VReg op_reg_class> :
Pat<(result_type (vop
(op_type op_reg_class:$rs1),
(op_type op_reg_class:$rs2))),
(!cast<Instruction>(instruction_name#"_VV_"# vlmul.MX#"_E"#sew)
op_reg_class:$rs1,
op_reg_class:$rs2,
avl, log2sew)>;

class VPatBinarySDNode_XI<SDPatternOperator vop,
string instruction_name,
string suffix,
Expand All @@ -107,6 +124,26 @@ class VPatBinarySDNode_XI<SDPatternOperator vop,
xop_kind:$rs2,
avl, sew)>;

class VPatBinarySDNode_XI_E<SDPatternOperator vop,
string instruction_name,
string suffix,
ValueType result_type,
ValueType vop_type,
int log2sew,
LMULInfo vlmul,
int sew,
OutPatFrag avl,
VReg vop_reg_class,
ComplexPattern SplatPatKind,
DAGOperand xop_kind> :
Pat<(result_type (vop
(vop_type vop_reg_class:$rs1),
(vop_type (SplatPatKind xop_kind:$rs2)))),
(!cast<Instruction>(instruction_name#_#suffix#_# vlmul.MX#"_E"#sew)
vop_reg_class:$rs1,
xop_kind:$rs2,
avl, log2sew)>;

multiclass VPatBinarySDNode_VV_VX<SDPatternOperator vop, string instruction_name> {
foreach vti = AllIntegerVectors in {
def : VPatBinarySDNode_VV<vop, instruction_name,
Expand All @@ -119,6 +156,19 @@ multiclass VPatBinarySDNode_VV_VX<SDPatternOperator vop, string instruction_name
}
}

multiclass VPatBinarySDNode_VV_VX_E<SDPatternOperator vop,
string instruction_name> {
foreach vti = AllIntegerVectors in {
def : VPatBinarySDNode_VV_E<vop, instruction_name,
vti.Vector, vti.Vector, vti.Log2SEW,
vti.LMul, vti.SEW, vti.AVL, vti.RegClass>;
def : VPatBinarySDNode_XI_E<vop, instruction_name, "VX",
vti.Vector, vti.Vector, vti.Log2SEW,
vti.LMul, vti.SEW, vti.AVL, vti.RegClass,
SplatPat, GPR>;
}
}

multiclass VPatBinarySDNode_VV_VX_VI<SDPatternOperator vop, string instruction_name,
Operand ImmType = simm5>
: VPatBinarySDNode_VV_VX<vop, instruction_name> {
Expand Down Expand Up @@ -148,6 +198,24 @@ class VPatBinarySDNode_VF<SDPatternOperator vop,
(xop_type xop_kind:$rs2),
avl, sew)>;

class VPatBinarySDNode_VF_E<SDPatternOperator vop,
string instruction_name,
ValueType result_type,
ValueType vop_type,
ValueType xop_type,
int log2sew,
LMULInfo vlmul,
int sew,
OutPatFrag avl,
VReg vop_reg_class,
DAGOperand xop_kind> :
Pat<(result_type (vop (vop_type vop_reg_class:$rs1),
(vop_type (SplatFPOp xop_kind:$rs2)))),
(!cast<Instruction>(instruction_name#"_"#vlmul.MX#"_E"#sew)
vop_reg_class:$rs1,
(xop_type xop_kind:$rs2),
avl, log2sew)>;

multiclass VPatBinaryFPSDNode_VV_VF<SDPatternOperator vop, string instruction_name> {
foreach vti = AllFloatVectors in {
def : VPatBinarySDNode_VV<vop, instruction_name,
Expand All @@ -160,6 +228,19 @@ multiclass VPatBinaryFPSDNode_VV_VF<SDPatternOperator vop, string instruction_na
}
}

multiclass VPatBinaryFPSDNode_VV_VF_E<SDPatternOperator vop,
string instruction_name> {
foreach vti = AllFloatVectors in {
def : VPatBinarySDNode_VV_E<vop, instruction_name,
vti.Vector, vti.Vector, vti.Log2SEW,
vti.LMul, vti.SEW, vti.AVL, vti.RegClass>;
def : VPatBinarySDNode_VF_E<vop, instruction_name#"_V"#vti.ScalarSuffix,
vti.Vector, vti.Vector, vti.Scalar,
vti.Log2SEW, vti.LMul, vti.SEW, vti.AVL,
vti.RegClass, vti.ScalarRegClass>;
}
}

multiclass VPatBinaryFPSDNode_R_VF<SDPatternOperator vop, string instruction_name> {
foreach fvti = AllFloatVectors in
def : Pat<(fvti.Vector (vop (fvti.Vector (SplatFPOp fvti.Scalar:$rs2)),
Expand All @@ -170,6 +251,17 @@ multiclass VPatBinaryFPSDNode_R_VF<SDPatternOperator vop, string instruction_nam
fvti.AVL, fvti.Log2SEW)>;
}

multiclass VPatBinaryFPSDNode_R_VF_E<SDPatternOperator vop,
string instruction_name> {
foreach fvti = AllFloatVectors in
def : Pat<(fvti.Vector (vop (fvti.Vector (SplatFPOp fvti.Scalar:$rs2)),
(fvti.Vector fvti.RegClass:$rs1))),
(!cast<Instruction>(instruction_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_E"#fvti.SEW)
fvti.RegClass:$rs1,
(fvti.Scalar fvti.ScalarRegClass:$rs2),
fvti.AVL, fvti.Log2SEW)>;
}

multiclass VPatIntegerSetCCSDNode_VV<string instruction_name,
CondCode cc> {
foreach vti = AllIntegerVectors in {
Expand Down Expand Up @@ -723,10 +815,10 @@ defm : VPatBinarySDNode_VV_VX<mulhs, "PseudoVMULH">;
defm : VPatBinarySDNode_VV_VX<mulhu, "PseudoVMULHU">;

// 11.11. Vector Integer Divide Instructions
defm : VPatBinarySDNode_VV_VX<udiv, "PseudoVDIVU">;
defm : VPatBinarySDNode_VV_VX<sdiv, "PseudoVDIV">;
defm : VPatBinarySDNode_VV_VX<urem, "PseudoVREMU">;
defm : VPatBinarySDNode_VV_VX<srem, "PseudoVREM">;
defm : VPatBinarySDNode_VV_VX_E<udiv, "PseudoVDIVU">;
defm : VPatBinarySDNode_VV_VX_E<sdiv, "PseudoVDIV">;
defm : VPatBinarySDNode_VV_VX_E<urem, "PseudoVREMU">;
defm : VPatBinarySDNode_VV_VX_E<srem, "PseudoVREM">;

// 11.12. Vector Widening Integer Multiply Instructions
defm : VPatWidenBinarySDNode_VV_VX<mul, sext_oneuse, sext_oneuse,
Expand Down Expand Up @@ -836,8 +928,8 @@ defm : VPatWidenBinaryFPSDNode_VV_VF_WV_WF<fsub, "PseudoVFWSUB">;

// 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
defm : VPatBinaryFPSDNode_VV_VF<any_fmul, "PseudoVFMUL">;
defm : VPatBinaryFPSDNode_VV_VF<any_fdiv, "PseudoVFDIV">;
defm : VPatBinaryFPSDNode_R_VF<any_fdiv, "PseudoVFRDIV">;
defm : VPatBinaryFPSDNode_VV_VF_E<any_fdiv, "PseudoVFDIV">;
defm : VPatBinaryFPSDNode_R_VF_E<any_fdiv, "PseudoVFRDIV">;

// 13.5. Vector Widening Floating-Point Multiply Instructions
defm : VPatWidenBinaryFPSDNode_VV_VF<fmul, "PseudoVFWMUL">;
Expand Down Expand Up @@ -914,7 +1006,7 @@ defm : VPatWidenFPNegMulSacSDNode_VV_VF<"PseudoVFWNMSAC">;
foreach vti = AllFloatVectors in {
// 13.8. Vector Floating-Point Square-Root Instruction
def : Pat<(fsqrt (vti.Vector vti.RegClass:$rs2)),
(!cast<Instruction>("PseudoVFSQRT_V_"# vti.LMul.MX)
(!cast<Instruction>("PseudoVFSQRT_V_"# vti.LMul.MX#"_E"#vti.SEW)
vti.RegClass:$rs2, vti.AVL, vti.Log2SEW)>;

// 13.12. Vector Floating-Point Sign-Injection Instructions
Expand Down
141 changes: 130 additions & 11 deletions llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,32 @@ multiclass VPatBinaryVL_V<SDPatternOperator vop,
(mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>;
}

multiclass VPatBinaryVL_V_E<SDPatternOperator vop,
string instruction_name,
string suffix,
ValueType result_type,
ValueType op1_type,
ValueType op2_type,
ValueType mask_type,
int log2sew,
LMULInfo vlmul,
int sew,
VReg result_reg_class,
VReg op1_reg_class,
VReg op2_reg_class> {
def : Pat<(result_type (vop
(op1_type op1_reg_class:$rs1),
(op2_type op2_reg_class:$rs2),
(result_type result_reg_class:$merge),
(mask_type V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#"_"#suffix#"_"# vlmul.MX#"_E"# sew#"_MASK")
result_reg_class:$merge,
op1_reg_class:$rs1,
op2_reg_class:$rs2,
(mask_type V0), GPR:$vl, log2sew, TAIL_AGNOSTIC)>;
}

multiclass VPatTiedBinaryNoMaskVL_V<SDNode vop,
string instruction_name,
string suffix,
Expand Down Expand Up @@ -524,6 +550,33 @@ multiclass VPatBinaryVL_XI<SDPatternOperator vop,
(mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>;
}

multiclass VPatBinaryVL_XI_E<SDPatternOperator vop,
string instruction_name,
string suffix,
ValueType result_type,
ValueType vop1_type,
ValueType vop2_type,
ValueType mask_type,
int log2sew,
LMULInfo vlmul,
int sew,
VReg result_reg_class,
VReg vop_reg_class,
ComplexPattern SplatPatKind,
DAGOperand xop_kind> {
def : Pat<(result_type (vop
(vop1_type vop_reg_class:$rs1),
(vop2_type (SplatPatKind (XLenVT xop_kind:$rs2))),
(result_type result_reg_class:$merge),
(mask_type V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#_#suffix#_# vlmul.MX#"_E"# sew#"_MASK")
result_reg_class:$merge,
vop_reg_class:$rs1,
xop_kind:$rs2,
(mask_type V0), GPR:$vl, log2sew, TAIL_AGNOSTIC)>;
}

multiclass VPatBinaryVL_VV_VX<SDPatternOperator vop, string instruction_name> {
foreach vti = AllIntegerVectors in {
defm : VPatBinaryVL_V<vop, instruction_name, "VV",
Expand All @@ -537,6 +590,20 @@ multiclass VPatBinaryVL_VV_VX<SDPatternOperator vop, string instruction_name> {
}
}

multiclass VPatBinaryVL_VV_VX_E<SDPatternOperator vop,
string instruction_name> {
foreach vti = AllIntegerVectors in {
defm : VPatBinaryVL_V_E<vop, instruction_name, "VV",
vti.Vector, vti.Vector, vti.Vector, vti.Mask,
vti.Log2SEW, vti.LMul, vti.SEW,
vti.RegClass, vti.RegClass, vti.RegClass>;
defm : VPatBinaryVL_XI_E<vop, instruction_name, "VX",
vti.Vector, vti.Vector, vti.Vector, vti.Mask,
vti.Log2SEW, vti.LMul, vti.SEW,
vti.RegClass, vti.RegClass, SplatPat, GPR>;
}
}

multiclass VPatBinaryVL_VV_VX_VI<SDPatternOperator vop, string instruction_name,
Operand ImmType = simm5>
: VPatBinaryVL_VV_VX<vop, instruction_name> {
Expand Down Expand Up @@ -625,6 +692,29 @@ multiclass VPatBinaryVL_VF<SDPatternOperator vop,
(mask_type V0), GPR:$vl, sew, TAIL_AGNOSTIC)>;
}

multiclass VPatBinaryVL_VF_E<SDPatternOperator vop,
string instruction_name,
ValueType result_type,
ValueType vop_type,
ValueType mask_type,
int log2sew,
LMULInfo vlmul,
int sew,
VReg result_reg_class,
VReg vop_reg_class,
RegisterClass scalar_reg_class> {
def : Pat<(result_type (vop (vop_type vop_reg_class:$rs1),
(vop_type (SplatFPOp scalar_reg_class:$rs2)),
(result_type result_reg_class:$merge),
(mask_type V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#"_"#vlmul.MX#"_E"#sew#"_MASK")
result_reg_class:$merge,
vop_reg_class:$rs1,
scalar_reg_class:$rs2,
(mask_type V0), GPR:$vl, log2sew, TAIL_AGNOSTIC)>;
}

multiclass VPatBinaryFPVL_VV_VF<SDPatternOperator vop, string instruction_name> {
foreach vti = AllFloatVectors in {
defm : VPatBinaryVL_V<vop, instruction_name, "VV",
Expand All @@ -638,6 +728,20 @@ multiclass VPatBinaryFPVL_VV_VF<SDPatternOperator vop, string instruction_name>
}
}

multiclass VPatBinaryFPVL_VV_VF_E<SDPatternOperator vop,
string instruction_name> {
foreach vti = AllFloatVectors in {
defm : VPatBinaryVL_V_E<vop, instruction_name, "VV",
vti.Vector, vti.Vector, vti.Vector, vti.Mask,
vti.Log2SEW, vti.LMul, vti.SEW,
vti.RegClass, vti.RegClass, vti.RegClass>;
defm : VPatBinaryVL_VF_E<vop, instruction_name#"_V"#vti.ScalarSuffix,
vti.Vector, vti.Vector, vti.Mask, vti.Log2SEW,
vti.LMul, vti.SEW, vti.RegClass, vti.RegClass,
vti.ScalarRegClass>;
}
}

multiclass VPatBinaryFPVL_R_VF<SDPatternOperator vop, string instruction_name> {
foreach fvti = AllFloatVectors in {
def : Pat<(fvti.Vector (vop (SplatFPOp fvti.ScalarRegClass:$rs2),
Expand All @@ -652,6 +756,21 @@ multiclass VPatBinaryFPVL_R_VF<SDPatternOperator vop, string instruction_name> {
}
}

multiclass VPatBinaryFPVL_R_VF_E<SDPatternOperator vop,
string instruction_name> {
foreach fvti = AllFloatVectors in {
def : Pat<(fvti.Vector (vop (SplatFPOp fvti.ScalarRegClass:$rs2),
fvti.RegClass:$rs1,
(fvti.Vector fvti.RegClass:$merge),
(fvti.Mask V0),
VLOpFrag)),
(!cast<Instruction>(instruction_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK")
fvti.RegClass:$merge,
fvti.RegClass:$rs1, fvti.ScalarRegClass:$rs2,
(fvti.Mask V0), GPR:$vl, fvti.Log2SEW, TAIL_AGNOSTIC)>;
}
}

multiclass VPatIntegerSetCCVL_VV<VTypeInfo vti, string instruction_name,
CondCode cc> {
def : Pat<(vti.Mask (riscv_setcc_vl (vti.Vector vti.RegClass:$rs1),
Expand Down Expand Up @@ -1511,10 +1630,10 @@ defm : VPatBinaryVL_VV_VX<riscv_mulhs_vl, "PseudoVMULH">;
defm : VPatBinaryVL_VV_VX<riscv_mulhu_vl, "PseudoVMULHU">;

// 11.11. Vector Integer Divide Instructions
defm : VPatBinaryVL_VV_VX<riscv_udiv_vl, "PseudoVDIVU">;
defm : VPatBinaryVL_VV_VX<riscv_sdiv_vl, "PseudoVDIV">;
defm : VPatBinaryVL_VV_VX<riscv_urem_vl, "PseudoVREMU">;
defm : VPatBinaryVL_VV_VX<riscv_srem_vl, "PseudoVREM">;
defm : VPatBinaryVL_VV_VX_E<riscv_udiv_vl, "PseudoVDIVU">;
defm : VPatBinaryVL_VV_VX_E<riscv_sdiv_vl, "PseudoVDIV">;
defm : VPatBinaryVL_VV_VX_E<riscv_urem_vl, "PseudoVREMU">;
defm : VPatBinaryVL_VV_VX_E<riscv_srem_vl, "PseudoVREM">;

// 11.12. Vector Widening Integer Multiply Instructions
defm : VPatBinaryWVL_VV_VX<riscv_vwmul_vl, "PseudoVWMUL">;
Expand Down Expand Up @@ -1640,8 +1759,8 @@ defm : VPatWidenBinaryFPVL_VV_VF_WV_WF<riscv_fsub_vl, "PseudoVFWSUB">;

// 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
defm : VPatBinaryFPVL_VV_VF<any_riscv_fmul_vl, "PseudoVFMUL">;
defm : VPatBinaryFPVL_VV_VF<any_riscv_fdiv_vl, "PseudoVFDIV">;
defm : VPatBinaryFPVL_R_VF<any_riscv_fdiv_vl, "PseudoVFRDIV">;
defm : VPatBinaryFPVL_VV_VF_E<any_riscv_fdiv_vl, "PseudoVFDIV">;
defm : VPatBinaryFPVL_R_VF_E<any_riscv_fdiv_vl, "PseudoVFRDIV">;

// 13.5. Vector Widening Floating-Point Multiply Instructions
defm : VPatWidenBinaryFPVL_VV_VF<riscv_fmul_vl, riscv_fpextend_vl_oneuse, "PseudoVFWMUL">;
Expand Down Expand Up @@ -1683,7 +1802,7 @@ foreach vti = AllFloatVectors in {
// 13.8. Vector Floating-Point Square-Root Instruction
def : Pat<(riscv_fsqrt_vl (vti.Vector vti.RegClass:$rs2), (vti.Mask V0),
VLOpFrag),
(!cast<Instruction>("PseudoVFSQRT_V_"# vti.LMul.MX #"_MASK")
(!cast<Instruction>("PseudoVFSQRT_V_"# vti.LMul.MX # "_E" # vti.SEW # "_MASK")
(vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2,
(vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>;

Expand Down Expand Up @@ -2047,7 +2166,7 @@ foreach vti = AllIntegerVectors in {
vti.RegClass:$merge,
(vti.Mask V0),
VLOpFrag)),
(!cast<Instruction>("PseudoVRGATHER_VV_"# vti.LMul.MX#"_MASK")
(!cast<Instruction>("PseudoVRGATHER_VV_"# vti.LMul.MX#"_E"# vti.SEW#"_MASK")
vti.RegClass:$merge, vti.RegClass:$rs2, vti.RegClass:$rs1,
(vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, GPR:$rs1,
Expand All @@ -2073,7 +2192,7 @@ foreach vti = AllIntegerVectors in {
if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then {
defvar emul_str = octuple_to_str<octuple_emul>.ret;
defvar ivti = !cast<VTypeInfo>("VI16" # emul_str);
defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_" # emul_str;
defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_E" # vti.SEW # "_" # emul_str;

def : Pat<(vti.Vector
(riscv_vrgatherei16_vv_vl vti.RegClass:$rs2,
Expand Down Expand Up @@ -2117,7 +2236,7 @@ foreach vti = AllFloatVectors in {
vti.RegClass:$merge,
(vti.Mask V0),
VLOpFrag)),
(!cast<Instruction>("PseudoVRGATHER_VV_"# vti.LMul.MX#"_MASK")
(!cast<Instruction>("PseudoVRGATHER_VV_"# vti.LMul.MX#"_E"# vti.SEW#"_MASK")
vti.RegClass:$merge, vti.RegClass:$rs2, vti.RegClass:$rs1,
(vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, GPR:$rs1,
Expand All @@ -2143,7 +2262,7 @@ foreach vti = AllFloatVectors in {
if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then {
defvar emul_str = octuple_to_str<octuple_emul>.ret;
defvar ivti = !cast<VTypeInfo>("VI16" # emul_str);
defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_" # emul_str;
defvar inst = "PseudoVRGATHEREI16_VV_" # vti.LMul.MX # "_E" # vti.SEW # "_" # emul_str;

def : Pat<(vti.Vector
(riscv_vrgatherei16_vv_vl vti.RegClass:$rs2,
Expand Down
131 changes: 97 additions & 34 deletions llvm/lib/Target/RISCV/RISCVScheduleV.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,19 @@ defvar SchedMxListFW = !listremove(SchedMxList, ["M8", "MF8"]);
// Used for widening floating-point Reduction as it doesn't contain MF8.
defvar SchedMxListFWRed = !listremove(SchedMxList, ["MF8"]);

class SchedSEWSet<string mx> {
list<int> val = !cond(!eq(mx, "M1"): [8, 16, 32, 64],
!eq(mx, "M2"): [8, 16, 32, 64],
!eq(mx, "M4"): [8, 16, 32, 64],
!eq(mx, "M8"): [8, 16, 32, 64],
!eq(mx, "MF2"): [8, 16, 32],
!eq(mx, "MF4"): [8, 16],
!eq(mx, "MF8"): [8]);
}

// Define multiclasses to define SchedWrite, SchedRead, WriteRes, and
// ReadAdvance for each (name, LMUL) pair for each LMUL in each of the
// SchedMxList variants above.

multiclass LMULSchedWritesImpl<string name, list<string> MxList> {
foreach mx = MxList in {
def name # "_" # mx : SchedWrite;
Expand All @@ -45,6 +54,46 @@ multiclass LMULReadAdvanceImpl<string name, int val,
}
}

// Define multiclasses to define SchedWrite, SchedRead, WriteRes, and
// ReadAdvance for each (name, LMUL, SEW) tuple for each LMUL in each of the
// SchedMxList variants above.
multiclass LMULSEWSchedWritesImpl<string name, list<string> MxList> {
foreach mx = MxList in {
if !eq(mx, "UpperBound") then
def name # "_" # mx : SchedWrite;
else
foreach sew = SchedSEWSet<mx>.val in
def name # "_" # mx # "_E" # sew : SchedWrite;
}
}
multiclass LMULSEWSchedReadsImpl<string name, list<string> MxList> {
foreach mx = MxList in {
if !eq(mx, "UpperBound") then
def name # "_" # mx : SchedRead;
else
foreach sew = SchedSEWSet<mx>.val in
def name # "_" # mx # "_E" # sew : SchedRead;
}
}
multiclass LMULSEWWriteResImpl<string name, list<ProcResourceKind> resources> {
foreach mx = SchedMxList in {
if !eq(mx, "UpperBound") then
def : WriteRes<!cast<SchedWrite>(name # "_" # mx), resources>;
else
foreach sew = SchedSEWSet<mx>.val in
def : WriteRes<!cast<SchedWrite>(name # "_" # mx # "_E" # sew), resources>;
}
}
multiclass LMULSEWReadAdvanceImpl<string name, int val,
list<SchedWrite> writes = []> {
foreach mx = SchedMxList in {
if !eq(mx, "UpperBound") then
def : ReadAdvance<!cast<SchedRead>(name # "_" # mx), val, writes>;
else
foreach sew = SchedSEWSet<mx>.val in
def : ReadAdvance<!cast<SchedRead>(name # "_" # mx # "_E" # sew), val, writes>;
}
}
// Define classes to define list containing all SchedWrites for each (name, LMUL)
// pair for each LMUL in each of the SchedMxList variants above and name in
// argument `names`. These classes can be used to construct a list of existing
Expand All @@ -69,6 +118,13 @@ multiclass LMULReadAdvance<string name, int val, list<SchedWrite> writes = []>
: LMULReadAdvanceImpl<name, val, writes>;
class LMULSchedWriteList<list<string> names> : LMULSchedWriteListImpl<names, SchedMxList>;

multiclass LMULSEWSchedWrites<string name> : LMULSEWSchedWritesImpl<name, SchedMxList>;
multiclass LMULSEWSchedReads<string name> : LMULSEWSchedReadsImpl<name, SchedMxList>;
multiclass LMULSEWWriteRes<string name, list<ProcResourceKind> resources>
: LMULSEWWriteResImpl<name, resources>;
multiclass LMULSEWReadAdvance<string name, int val, list<SchedWrite> writes = []>
: LMULSEWReadAdvanceImpl<name, val, writes>;

multiclass LMULSchedWritesW<string name> : LMULSchedWritesImpl<name, SchedMxListW>;
multiclass LMULSchedReadsW<string name> : LMULSchedReadsImpl<name, SchedMxListW>;
multiclass LMULWriteResW<string name, list<ProcResourceKind> resources>
Expand Down Expand Up @@ -191,8 +247,8 @@ defm "" : LMULSchedWrites<"WriteVICmpI">;
defm "" : LMULSchedWrites<"WriteVIMulV">;
defm "" : LMULSchedWrites<"WriteVIMulX">;
// 11.11. Vector Integer Divide Instructions
defm "" : LMULSchedWrites<"WriteVIDivV">;
defm "" : LMULSchedWrites<"WriteVIDivX">;
defm "" : LMULSEWSchedWrites<"WriteVIDivV">;
defm "" : LMULSEWSchedWrites<"WriteVIDivX">;
// 11.12. Vector Widening Integer Multiply Instructions
defm "" : LMULSchedWritesW<"WriteVIWMulV">;
defm "" : LMULSchedWritesW<"WriteVIWMulX">;
Expand Down Expand Up @@ -241,8 +297,8 @@ defm "" : LMULSchedWritesFW<"WriteVFWALUF">;
// 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
defm "" : LMULSchedWrites<"WriteVFMulV">;
defm "" : LMULSchedWrites<"WriteVFMulF">;
defm "" : LMULSchedWrites<"WriteVFDivV">;
defm "" : LMULSchedWrites<"WriteVFDivF">;
defm "" : LMULSEWSchedWrites<"WriteVFDivV">;
defm "" : LMULSEWSchedWrites<"WriteVFDivF">;
// 13.5. Vector Widening Floating-Point Multiply
defm "" : LMULSchedWritesFW<"WriteVFWMulV">;
defm "" : LMULSchedWritesFW<"WriteVFWMulF">;
Expand All @@ -253,7 +309,7 @@ defm "" : LMULSchedWrites<"WriteVFMulAddF">;
defm "" : LMULSchedWritesFW<"WriteVFWMulAddV">;
defm "" : LMULSchedWritesFW<"WriteVFWMulAddF">;
// 13.8. Vector Floating-Point Square-Root Instruction
defm "" : LMULSchedWrites<"WriteVFSqrtV">;
defm "" : LMULSEWSchedWrites<"WriteVFSqrtV">;
// 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction
// 13.10. Vector Floating-Point Reciprocal Estimate Instruction
defm "" : LMULSchedWrites<"WriteVFRecpV">;
Expand Down Expand Up @@ -327,11 +383,11 @@ defm "" : LMULSchedWrites<"WriteVISlideI">;
defm "" : LMULSchedWrites<"WriteVISlide1X">;
defm "" : LMULSchedWrites<"WriteVFSlide1F">;
// 16.4. Vector Register Gather Instructions
defm "" : LMULSchedWrites<"WriteVGatherV">;
defm "" : LMULSchedWrites<"WriteVGatherX">;
defm "" : LMULSchedWrites<"WriteVGatherI">;
defm "" : LMULSEWSchedWrites<"WriteVRGatherVV">;
defm "" : LMULSchedWrites<"WriteVRGatherVX">;
defm "" : LMULSchedWrites<"WriteVRGatherVI">;
// 16.5. Vector Compress Instruction
defm "" : LMULSchedWrites<"WriteVCompressV">;
defm "" : LMULSEWSchedWrites<"WriteVCompressV">;
// 16.6. Whole Vector Register Move
// These are already LMUL aware
def WriteVMov1V : SchedWrite;
Expand Down Expand Up @@ -415,8 +471,8 @@ defm "" : LMULSchedReads<"ReadVICmpX">;
defm "" : LMULSchedReads<"ReadVIMulV">;
defm "" : LMULSchedReads<"ReadVIMulX">;
// 11.11. Vector Integer Divide Instructions
defm "" : LMULSchedReads<"ReadVIDivV">;
defm "" : LMULSchedReads<"ReadVIDivX">;
defm "" : LMULSEWSchedReads<"ReadVIDivV">;
defm "" : LMULSEWSchedReads<"ReadVIDivX">;
// 11.12. Vector Widening Integer Multiply Instructions
defm "" : LMULSchedReadsW<"ReadVIWMulV">;
defm "" : LMULSchedReadsW<"ReadVIWMulX">;
Expand Down Expand Up @@ -460,8 +516,8 @@ defm "" : LMULSchedReadsFW<"ReadVFWALUF">;
// 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
defm "" : LMULSchedReads<"ReadVFMulV">;
defm "" : LMULSchedReads<"ReadVFMulF">;
defm "" : LMULSchedReads<"ReadVFDivV">;
defm "" : LMULSchedReads<"ReadVFDivF">;
defm "" : LMULSEWSchedReads<"ReadVFDivV">;
defm "" : LMULSEWSchedReads<"ReadVFDivF">;
// 13.5. Vector Widening Floating-Point Multiply
defm "" : LMULSchedReadsFW<"ReadVFWMulV">;
defm "" : LMULSchedReadsFW<"ReadVFWMulF">;
Expand All @@ -472,7 +528,7 @@ defm "" : LMULSchedReads<"ReadVFMulAddF">;
defm "" : LMULSchedReadsFW<"ReadVFWMulAddV">;
defm "" : LMULSchedReadsFW<"ReadVFWMulAddF">;
// 13.8. Vector Floating-Point Square-Root Instruction
defm "" : LMULSchedReads<"ReadVFSqrtV">;
defm "" : LMULSEWSchedReads<"ReadVFSqrtV">;
// 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction
// 13.10. Vector Floating-Point Reciprocal Estimate Instruction
defm "" : LMULSchedReads<"ReadVFRecpV">;
Expand Down Expand Up @@ -549,10 +605,13 @@ defm "" : LMULSchedReads<"ReadVISlideX">;
defm "" : LMULSchedReads<"ReadVFSlideV">;
defm "" : LMULSchedReads<"ReadVFSlideF">;
// 16.4. Vector Register Gather Instructions
defm "" : LMULSchedReads<"ReadVGatherV">;
defm "" : LMULSchedReads<"ReadVGatherX">;
defm "" : LMULSEWSchedReads<"ReadVRGatherVV_data">;
defm "" : LMULSEWSchedReads<"ReadVRGatherVV_index">;
defm "" : LMULSchedReads<"ReadVRGatherVX_data">;
defm "" : LMULSchedReads<"ReadVRGatherVX_index">;
defm "" : LMULSchedReads<"ReadVRGatherVI_data">;
// 16.5. Vector Compress Instruction
defm "" : LMULSchedReads<"ReadVCompressV">;
defm "" : LMULSEWSchedReads<"ReadVCompressV">;
// 16.6. Whole Vector Register Move
// These are already LMUL aware
def ReadVMov1V : SchedRead;
Expand Down Expand Up @@ -653,8 +712,8 @@ defm "" : LMULWriteRes<"WriteVICmpX", []>;
defm "" : LMULWriteRes<"WriteVICmpI", []>;
defm "" : LMULWriteRes<"WriteVIMulV", []>;
defm "" : LMULWriteRes<"WriteVIMulX", []>;
defm "" : LMULWriteRes<"WriteVIDivV", []>;
defm "" : LMULWriteRes<"WriteVIDivX", []>;
defm "" : LMULSEWWriteRes<"WriteVIDivV", []>;
defm "" : LMULSEWWriteRes<"WriteVIDivX", []>;
defm "" : LMULWriteResW<"WriteVIWMulV", []>;
defm "" : LMULWriteResW<"WriteVIWMulX", []>;
defm "" : LMULWriteRes<"WriteVIMulAddV", []>;
Expand Down Expand Up @@ -690,15 +749,15 @@ defm "" : LMULWriteResFW<"WriteVFWALUV", []>;
defm "" : LMULWriteResFW<"WriteVFWALUF", []>;
defm "" : LMULWriteRes<"WriteVFMulV", []>;
defm "" : LMULWriteRes<"WriteVFMulF", []>;
defm "" : LMULWriteRes<"WriteVFDivV", []>;
defm "" : LMULWriteRes<"WriteVFDivF", []>;
defm "" : LMULSEWWriteRes<"WriteVFDivV", []>;
defm "" : LMULSEWWriteRes<"WriteVFDivF", []>;
defm "" : LMULWriteResFW<"WriteVFWMulV", []>;
defm "" : LMULWriteResFW<"WriteVFWMulF", []>;
defm "" : LMULWriteRes<"WriteVFMulAddV", []>;
defm "" : LMULWriteRes<"WriteVFMulAddF", []>;
defm "" : LMULWriteResFW<"WriteVFWMulAddV", []>;
defm "" : LMULWriteResFW<"WriteVFWMulAddF", []>;
defm "" : LMULWriteRes<"WriteVFSqrtV", []>;
defm "" : LMULSEWWriteRes<"WriteVFSqrtV", []>;
defm "" : LMULWriteRes<"WriteVFRecpV", []>;
defm "" : LMULWriteRes<"WriteVFCmpV", []>;
defm "" : LMULWriteRes<"WriteVFCmpF", []>;
Expand Down Expand Up @@ -741,10 +800,10 @@ defm "" : LMULWriteRes<"WriteVISlideX", []>;
defm "" : LMULWriteRes<"WriteVISlideI", []>;
defm "" : LMULWriteRes<"WriteVISlide1X", []>;
defm "" : LMULWriteRes<"WriteVFSlide1F", []>;
defm "" : LMULWriteRes<"WriteVGatherV", []>;
defm "" : LMULWriteRes<"WriteVGatherX", []>;
defm "" : LMULWriteRes<"WriteVGatherI", []>;
defm "" : LMULWriteRes<"WriteVCompressV", []>;
defm "" : LMULSEWWriteRes<"WriteVRGatherVV", []>;
defm "" : LMULWriteRes<"WriteVRGatherVX", []>;
defm "" : LMULWriteRes<"WriteVRGatherVI", []>;
defm "" : LMULSEWWriteRes<"WriteVCompressV", []>;
// These are already LMUL aware
def : WriteRes<WriteVMov1V, []>;
def : WriteRes<WriteVMov2V, []>;
Expand Down Expand Up @@ -808,8 +867,8 @@ defm "" : LMULReadAdvance<"ReadVICmpV", 0>;
defm "" : LMULReadAdvance<"ReadVICmpX", 0>;
defm "" : LMULReadAdvance<"ReadVIMulV", 0>;
defm "" : LMULReadAdvance<"ReadVIMulX", 0>;
defm "" : LMULReadAdvance<"ReadVIDivV", 0>;
defm "" : LMULReadAdvance<"ReadVIDivX", 0>;
defm "" : LMULSEWReadAdvance<"ReadVIDivV", 0>;
defm "" : LMULSEWReadAdvance<"ReadVIDivX", 0>;
defm "" : LMULReadAdvanceW<"ReadVIWMulV", 0>;
defm "" : LMULReadAdvanceW<"ReadVIWMulX", 0>;
defm "" : LMULReadAdvance<"ReadVIMulAddV", 0>;
Expand Down Expand Up @@ -840,15 +899,15 @@ defm "" : LMULReadAdvanceFW<"ReadVFWALUV", 0>;
defm "" : LMULReadAdvanceFW<"ReadVFWALUF", 0>;
defm "" : LMULReadAdvance<"ReadVFMulV", 0>;
defm "" : LMULReadAdvance<"ReadVFMulF", 0>;
defm "" : LMULReadAdvance<"ReadVFDivV", 0>;
defm "" : LMULReadAdvance<"ReadVFDivF", 0>;
defm "" : LMULSEWReadAdvance<"ReadVFDivV", 0>;
defm "" : LMULSEWReadAdvance<"ReadVFDivF", 0>;
defm "" : LMULReadAdvanceFW<"ReadVFWMulV", 0>;
defm "" : LMULReadAdvanceFW<"ReadVFWMulF", 0>;
defm "" : LMULReadAdvance<"ReadVFMulAddV", 0>;
defm "" : LMULReadAdvance<"ReadVFMulAddF", 0>;
defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>;
defm "" : LMULReadAdvanceFW<"ReadVFWMulAddF", 0>;
defm "" : LMULReadAdvance<"ReadVFSqrtV", 0>;
defm "" : LMULSEWReadAdvance<"ReadVFSqrtV", 0>;
defm "" : LMULReadAdvance<"ReadVFRecpV", 0>;
defm "" : LMULReadAdvance<"ReadVFCmpV", 0>;
defm "" : LMULReadAdvance<"ReadVFCmpF", 0>;
Expand Down Expand Up @@ -899,9 +958,13 @@ defm "" : LMULReadAdvance<"ReadVISlideV", 0>;
defm "" : LMULReadAdvance<"ReadVISlideX", 0>;
defm "" : LMULReadAdvance<"ReadVFSlideV", 0>;
defm "" : LMULReadAdvance<"ReadVFSlideF", 0>;
defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_data", 0>;
defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_index", 0>;
defm "" : LMULReadAdvance<"ReadVRGatherVX_data", 0>;
defm "" : LMULReadAdvance<"ReadVRGatherVX_index", 0>;
defm "" : LMULReadAdvance<"ReadVRGatherVI_data", 0>;
defm "" : LMULReadAdvance<"ReadVGatherV", 0>;
defm "" : LMULReadAdvance<"ReadVGatherX", 0>;
defm "" : LMULReadAdvance<"ReadVCompressV", 0>;
defm "" : LMULSEWReadAdvance<"ReadVCompressV", 0>;
// These are already LMUL aware
def : ReadAdvance<ReadVMov1V, 0>;
def : ReadAdvance<ReadVMov2V, 0>;
Expand Down