diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td index b48db0e8163aa5..3d793d6660fefd 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td @@ -144,6 +144,8 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; // FIXME: Check MMO is atomic def : GINodeEquiv; diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index f491b6ac61d030..678f69873aec4a 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -2775,6 +2775,9 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI, case Intrinsic::amdgcn_raw_buffer_store_format: case Intrinsic::amdgcn_struct_buffer_store_format: return legalizeBufferStore(MI, MRI, B, false, true); + case Intrinsic::amdgcn_raw_tbuffer_store: + case Intrinsic::amdgcn_struct_tbuffer_store: + return legalizeBufferStore(MI, MRI, B, true, true); case Intrinsic::amdgcn_raw_buffer_load: case Intrinsic::amdgcn_struct_buffer_load: return legalizeBufferLoad(MI, MRI, B, false, false); diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp index e6fe02e1aca6e9..5ee80d93c3193b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp @@ -2256,7 +2256,9 @@ void AMDGPURegisterBankInfo::applyMappingImpl( case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE: case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT: case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT: - case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: { + case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: + case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT: + case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: { applyDefaultMapping(OpdMapper); executeInWaterfallLoop(MI, MRI, {1, 4}); return; diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index eeefa40ba38b3a..38d5aa4dbe841c 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -1760,8 +1760,8 @@ multiclass MTBUF_StoreIntrinsicPat(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, - (as_i16imm $offset), (as_i8imm $format), + (!cast(opcode # _OFFSET_exact) getVregSrcForVT.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, + (as_i16timm $offset), (as_i8timm $format), (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), (extract_swz $auxiliary)) >; @@ -1769,8 +1769,8 @@ multiclass MTBUF_StoreIntrinsicPat(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, - (as_i16imm $offset), (as_i8imm $format), + (!cast(opcode # _IDXEN_exact) getVregSrcForVT.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, + (as_i16timm $offset), (as_i8timm $format), (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), (extract_swz $auxiliary)) >; @@ -1778,8 +1778,8 @@ multiclass MTBUF_StoreIntrinsicPat(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, - (as_i16imm $offset), (as_i8imm $format), + (!cast(opcode # _OFFEN_exact) getVregSrcForVT.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, + (as_i16timm $offset), (as_i8timm $format), (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), (extract_swz $auxiliary)) >; @@ -1788,9 +1788,9 @@ multiclass MTBUF_StoreIntrinsicPat(opcode # _BOTHEN_exact) - $vdata, - (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), - $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format), + getVregSrcForVT.ret:$vdata, + (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), + SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), (extract_swz $auxiliary)) >;