diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 4b74f3b81e5e7..0636a74a61dc5 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -155,12 +155,12 @@ class MTBUF_Real : } class getMTBUFInsDA vdataList, - list vaddrList=[], bit hasGFX12Enc> { + list vaddrList=[], bit hasRestrictedSOffset> { RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); RegisterOperand vdata_op = getLdStRegisterOperand.ret; - dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset), + dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset, @@ -174,13 +174,13 @@ class getMTBUFInsDA vdataList, !con((ins vdata_op:$vdata), Inputs)); } -class getMTBUFIns vdataList=[], bit hasGFX12Enc> { +class getMTBUFIns vdataList=[], bit hasRestrictedSOffset> { dag ret = - !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA.ret, - !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA.ret, - !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA.ret, - !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA.ret, - !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA.ret, + !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA.ret, + !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA.ret, + !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA.ret, + !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA.ret, + !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA.ret, (ins)))))); } @@ -215,13 +215,13 @@ class MTBUF_Load_Pseudo pattern=[], // Workaround bug bz30254 int addrKindCopy = addrKind> : MTBUF_Pseudo.ret:$vdata), - getMTBUFIns.ret, + getMTBUFIns.ret, getMTBUFAsmOps.ret, pattern>, MTBUF_SetupAddr { @@ -232,23 +232,23 @@ class MTBUF_Load_Pseudo { + int elems, bit hasRestrictedSOffset> { - def _OFFSET : MTBUF_Load_Pseudo , + def _OFFSET : MTBUF_Load_Pseudo , MTBUFAddr64Table<0, NAME>; - def _ADDR64 : MTBUF_Load_Pseudo , + def _ADDR64 : MTBUF_Load_Pseudo , MTBUFAddr64Table<1, NAME>; - def _OFFEN : MTBUF_Load_Pseudo ; - def _IDXEN : MTBUF_Load_Pseudo ; - def _BOTHEN : MTBUF_Load_Pseudo ; + def _OFFEN : MTBUF_Load_Pseudo ; + def _IDXEN : MTBUF_Load_Pseudo ; + def _BOTHEN : MTBUF_Load_Pseudo ; let DisableWQM = 1 in { - def _OFFSET_exact : MTBUF_Load_Pseudo ; - def _OFFEN_exact : MTBUF_Load_Pseudo ; - def _IDXEN_exact : MTBUF_Load_Pseudo ; - def _BOTHEN_exact : MTBUF_Load_Pseudo ; + def _OFFSET_exact : MTBUF_Load_Pseudo ; + def _OFFEN_exact : MTBUF_Load_Pseudo ; + def _IDXEN_exact : MTBUF_Load_Pseudo ; + def _BOTHEN_exact : MTBUF_Load_Pseudo ; } } @@ -262,14 +262,14 @@ class MTBUF_Store_Pseudo pattern=[], // Workaround bug bz30254 int addrKindCopy = addrKind, RegisterClass vdataClassCopy = vdataClass> : MTBUF_Pseudo.ret, + getMTBUFIns.ret, getMTBUFAsmOps.ret, pattern>, MTBUF_SetupAddr { @@ -280,23 +280,23 @@ class MTBUF_Store_Pseudo { + int elems, bit hasRestrictedSOffset> { - def _OFFSET : MTBUF_Store_Pseudo , + def _OFFSET : MTBUF_Store_Pseudo , MTBUFAddr64Table<0, NAME>; - def _ADDR64 : MTBUF_Store_Pseudo , + def _ADDR64 : MTBUF_Store_Pseudo , MTBUFAddr64Table<1, NAME>; - def _OFFEN : MTBUF_Store_Pseudo ; - def _IDXEN : MTBUF_Store_Pseudo ; - def _BOTHEN : MTBUF_Store_Pseudo ; + def _OFFEN : MTBUF_Store_Pseudo ; + def _IDXEN : MTBUF_Store_Pseudo ; + def _BOTHEN : MTBUF_Store_Pseudo ; let DisableWQM = 1 in { - def _OFFSET_exact : MTBUF_Store_Pseudo ; - def _OFFEN_exact : MTBUF_Store_Pseudo ; - def _IDXEN_exact : MTBUF_Store_Pseudo ; - def _BOTHEN_exact : MTBUF_Store_Pseudo ; + def _OFFSET_exact : MTBUF_Store_Pseudo ; + def _OFFEN_exact : MTBUF_Store_Pseudo ; + def _IDXEN_exact : MTBUF_Store_Pseudo ; + def _BOTHEN_exact : MTBUF_Store_Pseudo ; } } @@ -405,12 +405,12 @@ class getLdStVDataRegisterOperand { } class getMUBUFInsDA vdataList, - list vaddrList, bit isTFE, bit hasGFX12Enc> { + list vaddrList, bit isTFE, bit hasRestrictedSOffset> { RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); RegisterOperand vdata_op = getLdStVDataRegisterOperand.ret; - dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); + dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset, (ins Offset:$offset, CPol_0:$cpol, i1imm_0:$swz)); dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs)); @@ -436,13 +436,13 @@ class getMUBUFElements { ); } -class getMUBUFIns vdataList, bit isTFE, bit hasGFX12Enc> { +class getMUBUFIns vdataList, bit isTFE, bit hasRestrictedSOffset> { dag ret = - !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA.ret, - !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA.ret, - !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA.ret, - !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA.ret, - !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA.ret, + !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA.ret, + !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA.ret, + !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA.ret, + !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA.ret, + !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA.ret, (ins)))))); } @@ -482,7 +482,7 @@ class MUBUF_Load_Pseudo pattern=[], // Workaround bug bz30254 int addrKindCopy = addrKind, @@ -490,7 +490,7 @@ class MUBUF_Load_Pseudo .ret> : MUBUF_Pseudo.ret, + !con(getMUBUFIns.ret, !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), getMUBUFAsmOps.ret, pattern>, @@ -536,24 +536,24 @@ multiclass MUBUF_Pseudo_Load_Pats { + bit TiedDest, bit isLds, bit isTFE, bit hasRestrictedSOffset> { defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); - def _OFFSET : MUBUF_Load_Pseudo , + def _OFFSET : MUBUF_Load_Pseudo , MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; - def _ADDR64 : MUBUF_Load_Pseudo , + def _ADDR64 : MUBUF_Load_Pseudo , MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; - def _OFFEN : MUBUF_Load_Pseudo ; - def _IDXEN : MUBUF_Load_Pseudo ; - def _BOTHEN : MUBUF_Load_Pseudo ; + def _OFFEN : MUBUF_Load_Pseudo ; + def _IDXEN : MUBUF_Load_Pseudo ; + def _BOTHEN : MUBUF_Load_Pseudo ; let DisableWQM = 1 in { - def _OFFSET_exact : MUBUF_Load_Pseudo ; - def _OFFEN_exact : MUBUF_Load_Pseudo ; - def _IDXEN_exact : MUBUF_Load_Pseudo ; - def _BOTHEN_exact : MUBUF_Load_Pseudo ; + def _OFFSET_exact : MUBUF_Load_Pseudo ; + def _OFFEN_exact : MUBUF_Load_Pseudo ; + def _IDXEN_exact : MUBUF_Load_Pseudo ; + def _BOTHEN_exact : MUBUF_Load_Pseudo ; } } @@ -596,13 +596,13 @@ class MUBUF_Store_Pseudo pattern=[], // Workaround bug bz30254 int addrKindCopy = addrKind> : MUBUF_Pseudo.ret.RegClass], isTFE, hasGFX12Enc>.ret, + getMUBUFIns.ret.RegClass], isTFE, hasRestrictedSOffset>.ret, getMUBUFAsmOps.ret, pattern>, MUBUF_SetupAddr { @@ -633,24 +633,24 @@ multiclass MUBUF_Pseudo_Store_Pats { + bit isTFE, bit hasRestrictedSOffset> { defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); - def _OFFSET : MUBUF_Store_Pseudo , + def _OFFSET : MUBUF_Store_Pseudo , MUBUFAddr64Table<0, NAME>; - def _ADDR64 : MUBUF_Store_Pseudo , + def _ADDR64 : MUBUF_Store_Pseudo , MUBUFAddr64Table<1, NAME>; - def _OFFEN : MUBUF_Store_Pseudo ; - def _IDXEN : MUBUF_Store_Pseudo ; - def _BOTHEN : MUBUF_Store_Pseudo ; + def _OFFEN : MUBUF_Store_Pseudo ; + def _IDXEN : MUBUF_Store_Pseudo ; + def _BOTHEN : MUBUF_Store_Pseudo ; let DisableWQM = 1 in { - def _OFFSET_exact : MUBUF_Store_Pseudo ; - def _OFFEN_exact : MUBUF_Store_Pseudo ; - def _IDXEN_exact : MUBUF_Store_Pseudo ; - def _BOTHEN_exact : MUBUF_Store_Pseudo ; + def _OFFSET_exact : MUBUF_Store_Pseudo ; + def _OFFEN_exact : MUBUF_Store_Pseudo ; + def _IDXEN_exact : MUBUF_Store_Pseudo ; + def _BOTHEN_exact : MUBUF_Store_Pseudo ; } } @@ -680,14 +680,14 @@ class MUBUF_Pseudo_Store_Lds let AsmMatchConverter = "cvtMubuf"; } -class getMUBUFAtomicInsDA vaddrList=[]> { RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); RegisterOperand vdata_op = getLdStRegisterOperand.ret; dag VData = !if(vdata_in, (ins vdata_op:$vdata_in), (ins vdata_op:$vdata)); dag Data = !if(!empty(vaddrList), VData, !con(VData, (ins vaddrClass:$vaddr))); - dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); + dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); dag MainInputs = !con((ins SReg_128:$srsrc), SOffset, (ins Offset:$offset)); dag CPol = !if(vdata_in, (ins CPol_GLC_WithDefault:$cpol), (ins CPol_NonGLC_WithDefault:$cpol)); @@ -698,20 +698,20 @@ class getMUBUFAtomicInsDA { dag ret = !if(!eq(addrKind, BUFAddrKind.Offset), - getMUBUFAtomicInsDA.ret, + getMUBUFAtomicInsDA.ret, !if(!eq(addrKind, BUFAddrKind.OffEn), - getMUBUFAtomicInsDA.ret, + getMUBUFAtomicInsDA.ret, !if(!eq(addrKind, BUFAddrKind.IdxEn), - getMUBUFAtomicInsDA.ret, + getMUBUFAtomicInsDA.ret, !if(!eq(addrKind, BUFAddrKind.BothEn), - getMUBUFAtomicInsDA.ret, + getMUBUFAtomicInsDA.ret, !if(!eq(addrKind, BUFAddrKind.Addr64), - getMUBUFAtomicInsDA.ret, + getMUBUFAtomicInsDA.ret, (ins)))))); } @@ -738,14 +738,14 @@ class MUBUF_Atomic_Pseudo pattern=[], // Workaround bug bz30254 int addrKindCopy = addrKind, RegisterClass vdataClassCopy = vdataClass> : MUBUF_Atomic_Pseudo.ret, + getMUBUFAtomicIns.ret, getMUBUFAsmOps.ret, pattern>, AtomicNoRet.ret, 0> { @@ -758,7 +758,7 @@ class MUBUF_AtomicNoRet_Pseudo pattern=[], // Workaround bug bz30254 int addrKindCopy = addrKind, @@ -766,7 +766,7 @@ class MUBUF_AtomicRet_Pseudo.ret> : MUBUF_Atomic_Pseudo.ret, + getMUBUFAtomicIns.ret, getMUBUFAsmOps.ret, pattern>, AtomicNoRet.ret, 1> {