diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index 9906049e9aaf9..5d8b091392bea 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -3762,36 +3762,45 @@ multiclass VPseudoVNCVTI_RM_W { multiclass VPseudoVNCVTF_W_RM { defvar constraint = "@earlyclobber $rd"; foreach m = MxListFW in { - defm _W : VPseudoConversionRoundingMode, - SchedUnary<"WriteVFNCvtIToFV", "ReadVFNCvtIToFV", m.MX, - forceMergeOpRead=true>; + foreach e = SchedSEWSet.val in + defm _W : VPseudoConversionRoundingMode, + SchedUnary<"WriteVFNCvtIToFV", "ReadVFNCvtIToFV", m.MX, e, + forceMergeOpRead=true>; } } multiclass VPseudoVNCVTF_RM_W { defvar constraint = "@earlyclobber $rd"; foreach m = MxListFW in { - defm _W : VPseudoConversionRM, - SchedUnary<"WriteVFNCvtIToFV", "ReadVFNCvtIToFV", m.MX, - forceMergeOpRead=true>; + foreach e = SchedSEWSet.val in + defm _W : VPseudoConversionRM, + SchedUnary<"WriteVFNCvtIToFV", "ReadVFNCvtIToFV", m.MX, e, + forceMergeOpRead=true>; } } multiclass VPseudoVNCVTD_W { defvar constraint = "@earlyclobber $rd"; foreach m = MxListFW in { - defm _W : VPseudoConversion, - SchedUnary<"WriteVFNCvtFToFV", "ReadVFNCvtFToFV", m.MX, - forceMergeOpRead=true>; + foreach e = SchedSEWSet.val in + defm _W : VPseudoConversion, + SchedUnary<"WriteVFNCvtFToFV", "ReadVFNCvtFToFV", m.MX, e, + forceMergeOpRead=true>; } } multiclass VPseudoVNCVTD_W_RM { defvar constraint = "@earlyclobber $rd"; foreach m = MxListFW in { - defm _W : VPseudoConversionRoundingMode, - SchedUnary<"WriteVFNCvtFToFV", "ReadVFNCvtFToFV", m.MX, - forceMergeOpRead=true>; + foreach e = SchedSEWSet.val in + defm _W : VPseudoConversionRoundingMode, + SchedUnary<"WriteVFNCvtFToFV", "ReadVFNCvtFToFV", m.MX, e, + forceMergeOpRead=true>; } } @@ -6027,7 +6036,8 @@ multiclass VPatConversionVI_WF_RM { } } -multiclass VPatConversionVF_WI_RM { +multiclass VPatConversionVF_WI_RM { foreach fvtiToFWti = AllWidenableFloatVectors in { defvar fvti = fvtiToFWti.Vti; defvar iwti = GetIntVTypeInfo.Vti; @@ -6035,11 +6045,13 @@ multiclass VPatConversionVF_WI_RM { GetVTypePredicates.Predicates) in defm : VPatConversionTARoundingMode; + fvti.LMul, fvti.RegClass, iwti.RegClass, + isSEWAware>; } } -multiclass VPatConversionVF_WF { +multiclass VPatConversionVF_WF { foreach fvtiToFWti = AllWidenableFloatVectors in { defvar fvti = fvtiToFWti.Vti; defvar fwti = fvtiToFWti.Wti; @@ -6047,12 +6059,13 @@ multiclass VPatConversionVF_WF { GetVTypePredicates.Predicates) in defm : VPatConversionTA; + fvti.LMul, fvti.RegClass, fwti.RegClass, isSEWAware>; } } -multiclass VPatConversionVF_WF_RM wlist = AllWidenableFloatVectors> { +multiclass VPatConversionVF_WF_RM wlist = AllWidenableFloatVectors, + bit isSEWAware = 0> { foreach fvtiToFWti = wlist in { defvar fvti = fvtiToFWti.Vti; defvar fwti = fvtiToFWti.Wti; @@ -6060,11 +6073,13 @@ multiclass VPatConversionVF_WF_RM .Predicates) in defm : VPatConversionTARoundingMode; + fvti.LMul, fvti.RegClass, fwti.RegClass, + isSEWAware>; } } -multiclass VPatConversionVF_WF_BF_RM { +multiclass VPatConversionVF_WF_BF_RM { foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in { defvar fvti = fvtiToFWti.Vti; defvar fwti = fvtiToFWti.Wti; @@ -6072,7 +6087,8 @@ multiclass VPatConversionVF_WF_BF_RM { GetVTypePredicates.Predicates) in defm : VPatConversionTARoundingMode; + fvti.LMul, fvti.RegClass, fwti.RegClass, + isSEWAware>; } } @@ -7320,21 +7336,24 @@ defm : VPatConversionVI_WF_RM<"int_riscv_vfncvt_xu_f_w", "PseudoVFNCVT_XU_F">; defm : VPatConversionVI_WF_RM<"int_riscv_vfncvt_x_f_w", "PseudoVFNCVT_X_F">; defm : VPatConversionVI_WF<"int_riscv_vfncvt_rtz_xu_f_w", "PseudoVFNCVT_RTZ_XU_F">; defm : VPatConversionVI_WF<"int_riscv_vfncvt_rtz_x_f_w", "PseudoVFNCVT_RTZ_X_F">; -defm : VPatConversionVF_WI_RM <"int_riscv_vfncvt_f_xu_w", "PseudoVFNCVT_F_XU">; -defm : VPatConversionVF_WI_RM <"int_riscv_vfncvt_f_x_w", "PseudoVFNCVT_F_X">; +defm : VPatConversionVF_WI_RM<"int_riscv_vfncvt_f_xu_w", "PseudoVFNCVT_F_XU", + isSEWAware=1>; +defm : VPatConversionVF_WI_RM<"int_riscv_vfncvt_f_x_w", "PseudoVFNCVT_F_X", + isSEWAware=1>; defvar WidenableFloatVectorsExceptF16 = !filter(fvtiToFWti, AllWidenableFloatVectors, !ne(fvtiToFWti.Vti.Scalar, f16)); defm : VPatConversionVF_WF_RM<"int_riscv_vfncvt_f_f_w", "PseudoVFNCVT_F_F", - WidenableFloatVectorsExceptF16>; + WidenableFloatVectorsExceptF16, isSEWAware=1>; // Define vfncvt.f.f.w for f16 when Zvfhmin is enable. defvar F16WidenableFloatVectors = !filter(fvtiToFWti, AllWidenableFloatVectors, !eq(fvtiToFWti.Vti.Scalar, f16)); let Predicates = [HasVInstructionsF16Minimal] in defm : VPatConversionVF_WF_RM<"int_riscv_vfncvt_f_f_w", "PseudoVFNCVT_F_F", - F16WidenableFloatVectors>; + F16WidenableFloatVectors, isSEWAware=1>; defm : VPatConversionVF_WF_BF_RM<"int_riscv_vfncvtbf16_f_f_w", - "PseudoVFNCVTBF16_F_F">; -defm : VPatConversionVF_WF<"int_riscv_vfncvt_rod_f_f_w", "PseudoVFNCVT_ROD_F_F">; + "PseudoVFNCVTBF16_F_F", isSEWAware=1>; +defm : VPatConversionVF_WF<"int_riscv_vfncvt_rod_f_f_w", "PseudoVFNCVT_ROD_F_F", + isSEWAware=1>; //===----------------------------------------------------------------------===// // 14. Vector Reduction Operations diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td index 23852414cf56b..9b324c40cc5be 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -470,7 +470,7 @@ multiclass VPatNConvertI2FPSDNode_W_RM.Predicates, GetVTypePredicates.Predicates) in def : Pat<(fvti.Vector (vop (iwti.Vector iwti.RegClass:$rs1))), - (!cast(instruction_name#"_"#fvti.LMul.MX) + (!cast(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW) (fvti.Vector (IMPLICIT_DEF)), iwti.RegClass:$rs1, // Value to indicate no rounding mode change in @@ -1445,7 +1445,7 @@ foreach fvtiToFWti = AllWidenableFloatVectors in { !listconcat(GetVTypePredicates.Predicates, GetVTypePredicates.Predicates)) in def : Pat<(fvti.Vector (fpround (fwti.Vector fwti.RegClass:$rs1))), - (!cast("PseudoVFNCVT_F_F_W_"#fvti.LMul.MX) + (!cast("PseudoVFNCVT_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW) (fvti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1, // Value to indicate no rounding mode change in diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td index eb46d47079b23..8e143b74a6663 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td @@ -1389,7 +1389,7 @@ multiclass VPatNConvertI2FPVL_W_RM(instruction_name#"_"#fvti.LMul.MX#"_MASK") + (!cast(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fvti.Vector (IMPLICIT_DEF)), iwti.RegClass:$rs1, (iwti.Mask V0), // Value to indicate no rounding mode change in @@ -1408,7 +1408,7 @@ multiclass VPatNConvertI2FP_RM_VL_W { def : Pat<(fvti.Vector (vop (iwti.Vector iwti.RegClass:$rs1), (iwti.Mask V0), (XLenVT timm:$frm), VLOpFrag)), - (!cast(instruction_name#"_"#fvti.LMul.MX#"_MASK") + (!cast(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fvti.Vector (IMPLICIT_DEF)), iwti.RegClass:$rs1, (iwti.Mask V0), timm:$frm, GPR:$vl, fvti.Log2SEW, TA_MA)>; } @@ -2703,7 +2703,7 @@ foreach fvtiToFWti = AllWidenableFloatVectors in { def : Pat<(fvti.Vector (any_riscv_fpround_vl (fwti.Vector fwti.RegClass:$rs1), (fwti.Mask V0), VLOpFrag)), - (!cast("PseudoVFNCVT_F_F_W_"#fvti.LMul.MX#"_MASK") + (!cast("PseudoVFNCVT_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fvti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1, (fwti.Mask V0), // Value to indicate no rounding mode change in @@ -2716,7 +2716,7 @@ foreach fvtiToFWti = AllWidenableFloatVectors in { def : Pat<(fvti.Vector (any_riscv_fncvt_rod_vl (fwti.Vector fwti.RegClass:$rs1), (fwti.Mask V0), VLOpFrag)), - (!cast("PseudoVFNCVT_ROD_F_F_W_"#fvti.LMul.MX#"_MASK") + (!cast("PseudoVFNCVT_ROD_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fvti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1, (fwti.Mask V0), GPR:$vl, fvti.Log2SEW, TA_MA)>; } diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td index f1246749ac433..feeefa7c42c49 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td +++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td @@ -819,11 +819,13 @@ foreach mx = SchedMxListW in { } } foreach mx = SchedMxListFW in { - defvar Cycles = SiFive7GetCyclesNarrowing.c; - defvar IsWorstCase = SiFive7IsWorstCaseMX.c; - let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in { - defm "" : LMULWriteResMX<"WriteVFNCvtIToFV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>; - defm "" : LMULWriteResMX<"WriteVFNCvtFToFV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>; + foreach sew = SchedSEWSet.val in { + defvar Cycles = SiFive7GetCyclesNarrowing.c; + defvar IsWorstCase = SiFive7IsWorstCaseMXSEW.c; + let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in { + defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtIToFV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>; + defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtFToFV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>; + } } } @@ -1182,9 +1184,9 @@ defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>; -defm "" : LMULReadAdvanceFW<"ReadVFNCvtIToFV", 0>; +defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtIToFV", 0>; defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>; -defm "" : LMULReadAdvanceFW<"ReadVFNCvtFToFV", 0>; +defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>; // 15. Vector Reduction Operations def : ReadAdvance; diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td index 0884cea396932..9dda9c500bd04 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td +++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td @@ -563,11 +563,13 @@ foreach mx = SchedMxListW in { } } foreach mx = SchedMxListFW in { - defvar LMulLat = SiFiveP600GetLMulCycles.c; - defvar IsWorstCase = SiFiveP600IsWorstCaseMX.c; - let Latency = 3, ReleaseAtCycles = [LMulLat] in { - defm "" : LMULWriteResMX<"WriteVFNCvtIToFV", [SiFiveP600VectorArith], mx, IsWorstCase>; - defm "" : LMULWriteResMX<"WriteVFNCvtFToFV", [SiFiveP600VectorArith], mx, IsWorstCase>; + foreach sew = SchedSEWSet.val in { + defvar LMulLat = SiFiveP600GetLMulCycles.c; + defvar IsWorstCase = SiFiveP600IsWorstCaseMXSEW.c; + let Latency = 3, ReleaseAtCycles = [LMulLat] in { + defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtIToFV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>; + defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtFToFV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>; + } } } @@ -965,9 +967,9 @@ defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>; -defm "" : LMULReadAdvanceFW<"ReadVFNCvtIToFV", 0>; +defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtIToFV", 0>; defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>; -defm "" : LMULReadAdvanceFW<"ReadVFNCvtFToFV", 0>; +defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>; // 15. Vector Reduction Operations def : ReadAdvance; diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td index 42c6dde89eac3..14a4e4cbc59f3 100644 --- a/llvm/lib/Target/RISCV/RISCVScheduleV.td +++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td @@ -468,9 +468,9 @@ defm "" : LMULSEWSchedWritesW<"WriteVFWCvtIToFV">; defm "" : LMULSchedWritesFW<"WriteVFWCvtFToIV">; defm "" : LMULSEWSchedWritesFW<"WriteVFWCvtFToFV">; // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions -defm "" : LMULSchedWritesFW<"WriteVFNCvtIToFV">; +defm "" : LMULSEWSchedWritesFW<"WriteVFNCvtIToFV">; defm "" : LMULSchedWritesW<"WriteVFNCvtFToIV">; -defm "" : LMULSchedWritesFW<"WriteVFNCvtFToFV">; +defm "" : LMULSEWSchedWritesFW<"WriteVFNCvtFToFV">; // 14. Vector Reduction Operations // The latency of reduction is determined by the size of the read resource. @@ -694,9 +694,9 @@ defm "" : LMULSEWSchedReadsW<"ReadVFWCvtIToFV">; defm "" : LMULSchedReadsFW<"ReadVFWCvtFToIV">; defm "" : LMULSEWSchedReadsFW<"ReadVFWCvtFToFV">; // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions -defm "" : LMULSchedReadsFW<"ReadVFNCvtIToFV">; +defm "" : LMULSEWSchedReadsFW<"ReadVFNCvtIToFV">; defm "" : LMULSchedReadsW<"ReadVFNCvtFToIV">; -defm "" : LMULSchedReadsFW<"ReadVFNCvtFToFV">; +defm "" : LMULSEWSchedReadsFW<"ReadVFNCvtFToFV">; // 14. Vector Reduction Operations // 14.1. Vector Single-Width Integer Reduction Instructions @@ -922,9 +922,9 @@ defm "" : LMULWriteRes<"WriteVFCvtFToIV", []>; defm "" : LMULSEWWriteResW<"WriteVFWCvtIToFV", []>; defm "" : LMULWriteResFW<"WriteVFWCvtFToIV", []>; defm "" : LMULSEWWriteResFW<"WriteVFWCvtFToFV", []>; -defm "" : LMULWriteResFW<"WriteVFNCvtIToFV", []>; +defm "" : LMULSEWWriteResFW<"WriteVFNCvtIToFV", []>; defm "" : LMULWriteResW<"WriteVFNCvtFToIV", []>; -defm "" : LMULWriteResFW<"WriteVFNCvtFToFV", []>; +defm "" : LMULSEWWriteResFW<"WriteVFNCvtFToFV", []>; // 14. Vector Reduction Operations defm "" : LMULSEWWriteRes<"WriteVIRedV_From", []>; @@ -1079,9 +1079,9 @@ defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>; -defm "" : LMULReadAdvanceFW<"ReadVFNCvtIToFV", 0>; +defm "" : LMULSEWReadAdvanceFW<"SEWReadVFNCvtIToFV", 0>; defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>; -defm "" : LMULReadAdvanceFW<"ReadVFNCvtFToFV", 0>; +defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>; // 14. Vector Reduction Operations def : ReadAdvance;