Skip to content

Commit

Permalink
[RISCV][CodeGen] Account for LMUL for Vector Integer load store instr…
Browse files Browse the repository at this point in the history 
…uctions

It is likley that subtargets act differently for a vector load store instructions based on the LMUL.
This patch creates seperate SchedRead, SchedWrite, WriteRes, ReadAdvance for each relevant LMUL.

Differential Revision: https://reviews.llvm.org/D137429
  • Loading branch information
@michaelmaitland
michaelmaitland committed Dec 7, 2022
1 parent 1a43227 commit 7b36502
Show file tree
Hide file tree
Showing 3 changed files with 322 additions and 271 deletions.
227 changes: 134 additions & 93 deletions llvm/lib/Target/RISCV/RISCVInstrInfoV.td
View file
Expand Up @@ -98,54 +98,91 @@ def simm5_plus1_nonzero : ImmLeaf<XLenVT,
// Scheduling definitions.
//===----------------------------------------------------------------------===//

class VMVRSched<int n>: Sched <[!cast<SchedReadWrite>("WriteVMov" # n # "V"),
!cast<SchedReadWrite>("ReadVMov" # n # "V")]>;

class VLESched : Sched <[WriteVLDE, ReadVLDX, ReadVMask]>;

class VSESched : Sched <[WriteVSTE, ReadVSTEV, ReadVSTX, ReadVMask]>;

class VLSSched<int n> : Sched <[!cast<SchedReadWrite>("WriteVLDS" # n),
ReadVLDX, ReadVLDSX, ReadVMask]>;

class VSSSched<int n> : Sched <[!cast<SchedReadWrite>("WriteVSTS" # n),
!cast<SchedReadWrite>("ReadVSTS" # n # "V"),
ReadVSTX, ReadVSTSX, ReadVMask]>;

class VLXSched<int n, string o> :
Sched <[!cast<SchedReadWrite>("WriteVLD" # o # "X" # n),
ReadVLDX, !cast<SchedReadWrite>("ReadVLD" # o # "XV"), ReadVMask]>;

class VSXSched<int n, string o> :
Sched <[!cast<SchedReadWrite>("WriteVST" # o # "X" # n),
!cast<SchedReadWrite>("ReadVST" # o # "X" # n),
ReadVSTX, !cast<SchedReadWrite>("ReadVST" # o # "XV"), ReadVMask]>;

class VLFSched : Sched <[WriteVLDFF, ReadVLDX, ReadVMask]>;
class VMVRSched<int n> : Sched<[
!cast<SchedReadWrite>("WriteVMov" #n #"V"),
!cast<SchedReadWrite>("ReadVMov" #n #"V")
]>;

class VLESched<string mx> : Sched<[
!cast<SchedReadWrite>("WriteVLDE_" #mx),
!cast<SchedReadWrite>("ReadVLDX_" #mx), ReadVMask
]>;

class VSESched<string mx> : Sched<[
!cast<SchedReadWrite>("WriteVSTE_" #mx),
!cast<SchedReadWrite>("ReadVSTEV_" #mx),
!cast<SchedReadWrite>("ReadVSTX_" #mx), ReadVMask
]>;

class VLSSched<int n, string mx> : Sched<[
!cast<SchedReadWrite>("WriteVLDS" #n #"_" #mx),
!cast<SchedReadWrite>("ReadVLDX_" #mx),
!cast<SchedReadWrite>("ReadVLDSX_" #mx), ReadVMask
]>;

class VSSSched<int n, string mx> : Sched<[
!cast<SchedReadWrite>("WriteVSTS" #n #"_" #mx),
!cast<SchedReadWrite>("ReadVSTS" #n #"V_" #mx),
!cast<SchedReadWrite>("ReadVSTX_" #mx),
!cast<SchedReadWrite>("ReadVSTSX_" #mx), ReadVMask
]>;

class VLXSched<int n, string o, string mx> : Sched<[
!cast<SchedReadWrite>("WriteVLD" #o #"X" #n #"_" #mx),
!cast<SchedReadWrite>("ReadVLDX_" #mx),
!cast<SchedReadWrite>("ReadVLD" #o #"XV_" #mx), ReadVMask
]>;

class VSXSched<int n, string o, string mx> : Sched<[
!cast<SchedReadWrite>("WriteVST" #o #"X" #n #"_" #mx),
!cast<SchedReadWrite>("ReadVST" #o #"X" #n #"_" #mx),
!cast<SchedReadWrite>("ReadVSTX_" #mx),
!cast<SchedReadWrite>("ReadVST" #o #"XV_" #mx), ReadVMask
]>;

class VLFSched<string mx> : Sched<[
!cast<SchedReadWrite>("WriteVLDFF_" #mx),
!cast<SchedReadWrite>("ReadVLDX_" #mx), ReadVMask
]>;

// Unit-Stride Segment Loads and Stores
class VLSEGSched<int nf, int eew> : Sched<[
!cast<SchedReadWrite>("WriteVLSEG" #nf #"e" #eew), ReadVLDX, ReadVMask]>;
class VSSEGSched<int nf, int eew> : Sched<[
!cast<SchedReadWrite>("WriteVSSEG" #nf #"e" #eew), ReadVSTEV, ReadVSTX,
ReadVMask]>;
class VLSEGFFSched<int nf, int eew> : Sched<[
!cast<SchedReadWrite>("WriteVLSEGFF" #nf #"e" #eew), ReadVLDX, ReadVMask]>;
class VLSEGSched<int nf, int eew, string mx> : Sched<[
!cast<SchedReadWrite>("WriteVLSEG" #nf #"e" #eew #"_" #mx),
!cast<SchedReadWrite>("ReadVLDX_" #mx), ReadVMask
]>;
class VSSEGSched<int nf, int eew, string mx> : Sched<[
!cast<SchedReadWrite>("WriteVSSEG" #nf #"e" #eew #"_" #mx),
!cast<SchedReadWrite>("ReadVSTEV_" #mx),
!cast<SchedReadWrite>("ReadVSTX_" #mx), ReadVMask
]>;
class VLSEGFFSched<int nf, int eew, string mx> : Sched<[
!cast<SchedReadWrite>("WriteVLSEGFF" #nf #"e" #eew #"_" #mx),
!cast<SchedReadWrite>("ReadVLDX_" #mx), ReadVMask
]>;
// Strided Segment Loads and Stores
class VLSSEGSched<int nf, int eew> : Sched<[
!cast<SchedReadWrite>("WriteVLSSEG" #nf #"e" #eew), ReadVLDX, ReadVLDSX,
ReadVMask]>;
class VSSSEGSched<int nf, int eew> : Sched<[
!cast<SchedReadWrite>("WriteVSSSEG" #nf #"e" #eew),
!cast<SchedReadWrite>("ReadVSTS" #eew #"V"), ReadVSTX, ReadVSTSX, ReadVMask]>;
class VLSSEGSched<int nf, int eew, string mx> : Sched<[
!cast<SchedReadWrite>("WriteVLSSEG" #nf #"e" #eew #"_" #mx),
!cast<SchedReadWrite>("ReadVLDX_" #mx),
!cast<SchedReadWrite>("ReadVLDSX_" #mx), ReadVMask
]>;
class VSSSEGSched<int nf, int eew, string mx> : Sched<[
!cast<SchedReadWrite>("WriteVSSSEG" #nf #"e" #eew #"_" #mx),
!cast<SchedReadWrite>("ReadVSTS" #eew #"V" #"_" #mx),
!cast<SchedReadWrite>("ReadVSTX_" #mx),
!cast<SchedReadWrite>("ReadVSTSX_" #mx), ReadVMask
]>;
// Indexed Segment Loads and Stores
class VLXSEGSched<int nf, int eew, string o> : Sched<[
!cast<SchedReadWrite>("WriteVL" #o # "XSEG" #nf #"e" #eew), ReadVLDX,
!cast<SchedReadWrite>("ReadVLD" # o # "XV"), ReadVMask]>;
class VSXSEGSched<int nf, int eew, string o> : Sched<[
!cast<SchedReadWrite>("WriteVS" #o # "XSEG" #nf #"e" #eew),
!cast<SchedReadWrite>("ReadVST" #o # "X" #eew), ReadVSTX,
!cast<SchedReadWrite>("ReadVST" #o # "XV"), ReadVMask]>;
class VLXSEGSched<int nf, int eew, string o, string mx> : Sched<[
!cast<SchedReadWrite>("WriteVL" #o #"XSEG" #nf #"e" #eew #"_" #mx),
!cast<SchedReadWrite>("ReadVLDX_" #mx),
!cast<SchedReadWrite>("ReadVLD" #o #"XV" #"_" #mx), ReadVMask
]>;
class VSXSEGSched<int nf, int eew, string o, string mx> : Sched<[
!cast<SchedReadWrite>("WriteVS" #o #"XSEG" #nf #"e" #eew #"_" #mx),
!cast<SchedReadWrite>("ReadVST" #o #"X" #eew # "_" # mx),
!cast<SchedReadWrite>("ReadVSTX_" #mx),
!cast<SchedReadWrite>("ReadVST" #o #"XV" # "_" # mx), ReadVMask
]>;

//===----------------------------------------------------------------------===//
// Instruction class templates
Expand Down Expand Up @@ -386,17 +423,17 @@ multiclass VIndexLoadStore<list<int> EEWList> {

def VLUXEI # n # _V :
VIndexedLoad<MOPLDIndexedUnord, w, "vluxei" # n # ".v">,
VLXSched<n, "U">;
VLXSched<n, "U", UpperBoundLMUL>;
def VLOXEI # n # _V :
VIndexedLoad<MOPLDIndexedOrder, w, "vloxei" # n # ".v">,
VLXSched<n, "O">;
VLXSched<n, "O", UpperBoundLMUL>;

def VSUXEI # n # _V :
VIndexedStore<MOPSTIndexedUnord, w, "vsuxei" # n # ".v">,
VSXSched<n, "U">;
VSXSched<n, "U", UpperBoundLMUL>;
def VSOXEI # n # _V :
VIndexedStore<MOPSTIndexedOrder, w, "vsoxei" # n # ".v">,
VSXSched<n, "O">;
VSXSched<n, "O", UpperBoundLMUL>;
}
}

Expand Down Expand Up @@ -921,12 +958,12 @@ multiclass VWholeLoadN<bits<3> nf, string opcodestr, RegisterClass VRC> {
defvar s = !cast<SchedWrite>("WriteVLD" # !add(nf, 1) # "R");

def E # l # _V : VWholeLoad<nf, w, opcodestr # "e" # l # ".v", VRC>,
Sched<[s, ReadVLDX]>;
Sched<[s, ReadVLDX_UpperBound]>;
}
}
multiclass VWholeLoadEEW64<bits<3> nf, string opcodestr, RegisterClass VRC, SchedReadWrite schedrw> {
def E64_V : VWholeLoad<nf, LSWidth64, opcodestr # "e64.v", VRC>,
Sched<[schedrw, ReadVLDX]>;
Sched<[schedrw, ReadVLDX_UpperBound]>;
}

//===----------------------------------------------------------------------===//
Expand All @@ -950,25 +987,25 @@ foreach eew = [8, 16, 32] in {
defvar w = !cast<RISCVWidth>("LSWidth" # eew);

// Vector Unit-Stride Instructions
def VLE#eew#_V : VUnitStrideLoad<w, "vle"#eew#".v">, VLESched;
def VSE#eew#_V : VUnitStrideStore<w, "vse"#eew#".v">, VSESched;
def VLE#eew#_V : VUnitStrideLoad<w, "vle"#eew#".v">, VLESched<UpperBoundLMUL>;
def VSE#eew#_V : VUnitStrideStore<w, "vse"#eew#".v">, VSESched<UpperBoundLMUL>;

// Vector Unit-Stride Fault-only-First Loads
def VLE#eew#FF_V : VUnitStrideLoadFF<w, "vle"#eew#"ff.v">, VLFSched;
def VLE#eew#FF_V : VUnitStrideLoadFF<w, "vle"#eew#"ff.v">, VLFSched<UpperBoundLMUL>;

// Vector Strided Instructions
def VLSE#eew#_V : VStridedLoad<w, "vlse"#eew#".v">, VLSSched<eew>;
def VSSE#eew#_V : VStridedStore<w, "vsse"#eew#".v">, VSSSched<eew>;
def VLSE#eew#_V : VStridedLoad<w, "vlse"#eew#".v">, VLSSched<eew, UpperBoundLMUL>;
def VSSE#eew#_V : VStridedStore<w, "vsse"#eew#".v">, VSSSched<eew, UpperBoundLMUL>;
}

defm "" : VIndexLoadStore<[8, 16, 32]>;
} // Predicates = [HasVInstructions]

let Predicates = [HasVInstructions] in {
def VLM_V : VUnitStrideLoadMask<"vlm.v">,
Sched<[WriteVLDM, ReadVLDX]>;
Sched<[WriteVLDM_UpperBound, ReadVLDX_UpperBound]>;
def VSM_V : VUnitStrideStoreMask<"vsm.v">,
Sched<[WriteVSTM, ReadVSTM, ReadVSTX]>;
Sched<[WriteVSTM_UpperBound, ReadVSTM_UpperBound, ReadVSTX_UpperBound]>;
def : InstAlias<"vle1.v $vd, (${rs1})",
(VLM_V VR:$vd, GPR:$rs1), 0>;
def : InstAlias<"vse1.v $vs3, (${rs1})",
Expand All @@ -980,13 +1017,13 @@ defm VL4R : VWholeLoadN<3, "vl4r", VRM4>;
defm VL8R : VWholeLoadN<7, "vl8r", VRM8>;

def VS1R_V : VWholeStore<0, "vs1r.v", VR>,
Sched<[WriteVST1R, ReadVST1R, ReadVSTX]>;
Sched<[WriteVST1R, ReadVST1R, ReadVSTX_UpperBound]>;
def VS2R_V : VWholeStore<1, "vs2r.v", VRM2>,
Sched<[WriteVST2R, ReadVST2R, ReadVSTX]>;
Sched<[WriteVST2R, ReadVST2R, ReadVSTX_UpperBound]>;
def VS4R_V : VWholeStore<3, "vs4r.v", VRM4>,
Sched<[WriteVST4R, ReadVST4R, ReadVSTX]>;
Sched<[WriteVST4R, ReadVST4R, ReadVSTX_UpperBound]>;
def VS8R_V : VWholeStore<7, "vs8r.v", VRM8>,
Sched<[WriteVST8R, ReadVST8R, ReadVSTX]>;
Sched<[WriteVST8R, ReadVST8R, ReadVSTX_UpperBound]>;

def : InstAlias<"vl1r.v $vd, (${rs1})", (VL1RE8_V VR:$vd, GPR:$rs1)>;
def : InstAlias<"vl2r.v $vd, (${rs1})", (VL2RE8_V VRM2:$vd, GPR:$rs1)>;
Expand All @@ -997,19 +1034,19 @@ def : InstAlias<"vl8r.v $vd, (${rs1})", (VL8RE8_V VRM8:$vd, GPR:$rs1)>;
let Predicates = [HasVInstructionsI64] in {
// Vector Unit-Stride Instructions
def VLE64_V : VUnitStrideLoad<LSWidth64, "vle64.v">,
VLESched;
VLESched<UpperBoundLMUL>;

def VLE64FF_V : VUnitStrideLoadFF<LSWidth64, "vle64ff.v">,
VLFSched;
VLFSched<UpperBoundLMUL>;

def VSE64_V : VUnitStrideStore<LSWidth64, "vse64.v">,
VSESched;
VSESched<UpperBoundLMUL>;
// Vector Strided Instructions
def VLSE64_V : VStridedLoad<LSWidth64, "vlse64.v">,
VLSSched<32>;
VLSSched<32, UpperBoundLMUL>;

def VSSE64_V : VStridedStore<LSWidth64, "vsse64.v">,
VSSSched<64>;
VSSSched<64, UpperBoundLMUL>;

defm VL1R: VWholeLoadEEW64<0, "vl1r", VR, WriteVLD1R>;
defm VL2R: VWholeLoadEEW64<1, "vl2r", VRM2, WriteVLD2R>;
Expand Down Expand Up @@ -1650,38 +1687,38 @@ let Predicates = [HasVInstructions] in {

def VLSEG#nf#E#eew#_V :
VUnitStrideSegmentLoad<!add(nf, -1), w, "vlseg"#nf#"e"#eew#".v">,
VLSEGSched<nf, eew>;
VLSEGSched<nf, eew, UpperBoundLMUL>;
def VLSEG#nf#E#eew#FF_V :
VUnitStrideSegmentLoadFF<!add(nf, -1), w, "vlseg"#nf#"e"#eew#"ff.v">,
VLSEGFFSched<nf, eew>;
VLSEGFFSched<nf, eew, UpperBoundLMUL>;
def VSSEG#nf#E#eew#_V :
VUnitStrideSegmentStore<!add(nf, -1), w, "vsseg"#nf#"e"#eew#".v">,
VSSEGSched<nf, eew>;
VSSEGSched<nf, eew, UpperBoundLMUL>;
// Vector Strided Instructions
def VLSSEG#nf#E#eew#_V :
VStridedSegmentLoad<!add(nf, -1), w, "vlsseg"#nf#"e"#eew#".v">,
VLSSEGSched<nf, eew>;
VLSSEGSched<nf, eew, UpperBoundLMUL>;
def VSSSEG#nf#E#eew#_V :
VStridedSegmentStore<!add(nf, -1), w, "vssseg"#nf#"e"#eew#".v">,
VSSSEGSched<nf, eew>;
VSSSEGSched<nf, eew, UpperBoundLMUL>;

// Vector Indexed Instructions
def VLUXSEG#nf#EI#eew#_V :
VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedUnord, w,
"vluxseg"#nf#"ei"#eew#".v">,
VLXSEGSched<nf, eew, "U">;
VLXSEGSched<nf, eew, "U", UpperBoundLMUL>;
def VLOXSEG#nf#EI#eew#_V :
VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedOrder, w,
"vloxseg"#nf#"ei"#eew#".v">,
VLXSEGSched<nf, eew, "O">;
VLXSEGSched<nf, eew, "O", UpperBoundLMUL>;
def VSUXSEG#nf#EI#eew#_V :
VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedUnord, w,
"vsuxseg"#nf#"ei"#eew#".v">,
VSXSEGSched<nf, eew, "U">;
VSXSEGSched<nf, eew, "U", UpperBoundLMUL>;
def VSOXSEG#nf#EI#eew#_V :
VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedOrder, w,
"vsoxseg"#nf#"ei"#eew#".v">,
VSXSEGSched<nf, eew, "O">;
VSXSEGSched<nf, eew, "O", UpperBoundLMUL>;
}
}
} // Predicates = [HasVInstructions]
Expand All @@ -1691,38 +1728,42 @@ let Predicates = [HasVInstructionsI64] in {
// Vector Unit-strided Segment Instructions
def VLSEG#nf#E64_V :
VUnitStrideSegmentLoad<!add(nf, -1), LSWidth64, "vlseg"#nf#"e64.v">,
VLSEGSched<nf, 64>;
VLSEGSched<nf, 64, UpperBoundLMUL>;
def VLSEG#nf#E64FF_V :
VUnitStrideSegmentLoadFF<!add(nf, -1), LSWidth64, "vlseg"#nf#"e64ff.v">,
VLSEGFFSched<nf, 64>;
VLSEGFFSched<nf, 64, UpperBoundLMUL>;
def VSSEG#nf#E64_V :
VUnitStrideSegmentStore<!add(nf, -1), LSWidth64, "vsseg"#nf#"e64.v">,
VSSEGSched<nf, 64>;
VSSEGSched<nf, 64, UpperBoundLMUL>;

// Vector Strided Segment Instructions
def VLSSEG#nf#E64_V :
VStridedSegmentLoad<!add(nf, -1), LSWidth64, "vlsseg"#nf#"e64.v">,
VLSSEGSched<nf, 64>;
VLSSEGSched<nf, 64, UpperBoundLMUL>;
def VSSSEG#nf#E64_V :
VStridedSegmentStore<!add(nf, -1), LSWidth64, "vssseg"#nf#"e64.v">,
VSSSEGSched<nf, 64>;
VSSSEGSched<nf, 64, UpperBoundLMUL>;
}
} // Predicates = [HasVInstructionsI64]
let Predicates = [HasVInstructionsI64, IsRV64] in {
foreach nf=2-8 in {
foreach nf = 2 - 8 in {
// Vector Indexed Segment Instructions
def VLUXSEG#nf#EI64_V :
VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedUnord, LSWidth64,
"vluxseg"#nf#"ei64.v">, VLXSEGSched<nf, 64, "U">;
def VLOXSEG#nf#EI64_V :
VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedOrder, LSWidth64,
"vloxseg"#nf#"ei64.v">, VLXSEGSched<nf, 64, "O">;
def VSUXSEG#nf#EI64_V :
VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedUnord, LSWidth64,
"vsuxseg"#nf#"ei64.v">, VSXSEGSched<nf, 64, "U">;
def VSOXSEG#nf#EI64_V :
VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedOrder, LSWidth64,
"vsoxseg"#nf#"ei64.v">, VSXSEGSched<nf, 64, "O">;
def VLUXSEG #nf #EI64_V
: VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedUnord, LSWidth64,
"vluxseg" #nf #"ei64.v">,
VLXSEGSched<nf, 64, "U", UpperBoundLMUL>;
def VLOXSEG #nf #EI64_V
: VIndexedSegmentLoad<!add(nf, -1), MOPLDIndexedOrder, LSWidth64,
"vloxseg" #nf #"ei64.v">,
VLXSEGSched<nf, 64, "O", UpperBoundLMUL>;
def VSUXSEG #nf #EI64_V
: VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedUnord, LSWidth64,
"vsuxseg" #nf #"ei64.v">,
VSXSEGSched<nf, 64, "U", UpperBoundLMUL>;
def VSOXSEG #nf #EI64_V
: VIndexedSegmentStore<!add(nf, -1), MOPSTIndexedOrder, LSWidth64,
"vsoxseg" #nf #"ei64.v">,
VSXSEGSched<nf, 64, "O", UpperBoundLMUL>;
}
} // Predicates = [HasVInstructionsI64, IsRV64]

Expand Down

0 comments on commit 7b36502

Please sign in to comment.