Skip to content

Add HVX vgather/vscatter Support #164421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 24, 2025
+1,374 −77
Merged

Add HVX vgather/vscatter Support #164421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions llvm/include/llvm/IR/IntrinsicsHexagon.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -447,3 +447,15 @@ def int_hexagon_instrprof_custom


include "llvm/IR/IntrinsicsHexagonDep.td"

class Hexagon__ptri32i32v64i16_Intrinsic<string GCCIntSuffix,
list<IntrinsicProperty> intr_properties = [IntrNoMem]>
: Hexagon_Intrinsic<GCCIntSuffix,
[], [llvm_ptr_ty,llvm_i32_ty,llvm_i32_ty,llvm_v64i16_ty],
intr_properties>;

def int_hexagon_V6_vgather_vscattermh :
Hexagon__ptri32i32v64i16_Intrinsic<"HEXAGON_V6_vgather_vscattermh", [IntrArgMemOnly]>;

def int_hexagon_V6_vgather_vscattermh_128B :
Hexagon__ptri32i32v32i32_Intrinsic<"HEXAGON_V6_vgather_vscattermh_128B", [IntrArgMemOnly]>;
4 changes: 3 additions & 1 deletion llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -654,7 +654,9 @@ void HexagonDAGToDAGISel::SelectIntrinsicWChain(SDNode *N) {
IntNo == Intrinsic::hexagon_V6_vgathermh ||
IntNo == Intrinsic::hexagon_V6_vgathermh_128B ||
IntNo == Intrinsic::hexagon_V6_vgathermhw ||
IntNo == Intrinsic::hexagon_V6_vgathermhw_128B) {
IntNo == Intrinsic::hexagon_V6_vgathermhw_128B ||
IntNo == Intrinsic::hexagon_V6_vgather_vscattermh ||
IntNo == Intrinsic::hexagon_V6_vgather_vscattermh_128B) {
SelectV65Gather(N);
return;
}
Expand Down
4 changes: 4 additions & 0 deletions llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2953,6 +2953,10 @@ void HexagonDAGToDAGISel::SelectV65Gather(SDNode *N) {
case Intrinsic::hexagon_V6_vgathermhw_128B:
Opcode = Hexagon::V6_vgathermhw_pseudo;
break;
case Intrinsic::hexagon_V6_vgather_vscattermh:
case Intrinsic::hexagon_V6_vgather_vscattermh_128B:
Opcode = Hexagon::V6_vgather_vscatter_mh_pseudo;
break;
}

SDVTList VTs = CurDAG->getVTList(MVT::Other);
Expand Down
4 changes: 3 additions & 1 deletion llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2145,7 +2145,9 @@ bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
case Intrinsic::hexagon_V6_vgathermhq:
case Intrinsic::hexagon_V6_vgathermhq_128B:
case Intrinsic::hexagon_V6_vgathermhwq:
case Intrinsic::hexagon_V6_vgathermhwq_128B: {
case Intrinsic::hexagon_V6_vgathermhwq_128B:
case Intrinsic::hexagon_V6_vgather_vscattermh:
case Intrinsic::hexagon_V6_vgather_vscattermh_128B: {
const Module &M = *I.getParent()->getParent()->getParent();
Info.opc = ISD::INTRINSIC_W_CHAIN;
Type *VecTy = I.getArgOperand(1)->getType();
Expand Down
162 changes: 88 additions & 74 deletions llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1554,80 +1554,93 @@ HexagonInstrInfo::expandVGatherPseudo(MachineInstr &MI) const {
MachineBasicBlock::iterator First;

switch (Opc) {
case Hexagon::V6_vgathermh_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermh))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();

case Hexagon::V6_vgathermw_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermw))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();

case Hexagon::V6_vgathermhw_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhw))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();

case Hexagon::V6_vgathermhq_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhq))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4))
.add(MI.getOperand(5));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();

case Hexagon::V6_vgathermwq_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermwq))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4))
.add(MI.getOperand(5));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();

case Hexagon::V6_vgathermhwq_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhwq))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4))
.add(MI.getOperand(5));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();
case Hexagon::V6_vgather_vscatter_mh_pseudo:
// This is mainly a place holder. It will be extended.
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermh))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vscattermh))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4))
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();
case Hexagon::V6_vgathermh_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermh))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();

case Hexagon::V6_vgathermw_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermw))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();

case Hexagon::V6_vgathermhw_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhw))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();

case Hexagon::V6_vgathermhq_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhq))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4))
.add(MI.getOperand(5));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();

case Hexagon::V6_vgathermwq_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermwq))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4))
.add(MI.getOperand(5));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();

case Hexagon::V6_vgathermhwq_pseudo:
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhwq))
.add(MI.getOperand(2))
.add(MI.getOperand(3))
.add(MI.getOperand(4))
.add(MI.getOperand(5));
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
.add(MI.getOperand(0))
.addImm(MI.getOperand(1).getImm())
.addReg(Hexagon::VTMP);
MBB.erase(MI);
return First.getInstrIterator();
}

return MI.getIterator();
Expand Down Expand Up @@ -2806,6 +2819,7 @@ bool HexagonInstrInfo::isValidOffset(unsigned Opcode, int Offset,
case Hexagon::V6_vL32b_nt_tmp_npred_ai:
case Hexagon::V6_vS32Ub_npred_ai:
case Hexagon::V6_vgathermh_pseudo:
case Hexagon::V6_vgather_vscatter_mh_pseudo:
case Hexagon::V6_vgathermw_pseudo:
case Hexagon::V6_vgathermhw_pseudo:
case Hexagon::V6_vgathermhq_pseudo:
Expand Down
13 changes: 13 additions & 0 deletions llvm/lib/Target/Hexagon/HexagonPatternsV65.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,19 @@ defm V6_vgathermh_pseudo : vgathermh<HvxVR>;
defm V6_vgathermw_pseudo : vgathermw<HvxVR>;
defm V6_vgathermhw_pseudo : vgathermhw<HvxWR>;


multiclass vgather_scatter_mh<RegisterClass RC> {
let isCodeGenOnly = 1, isPseudo = 1, mayLoad = 1,
mayStore = 1, addrMode = BaseImmOffset, accessSize = HalfWordAccess in
def NAME : CVI_GATHER_TMP_LD_Resource_NoOpcode<(outs ),
(ins IntRegs:$_dst_, s4_0Imm:$Ii,
IntRegs:$Rt, ModRegs:$Mu, RC:$Vv),
".error \"should not emit\" ",
[]>;
}

defm V6_vgather_vscatter_mh_pseudo : vgather_scatter_mh<HvxVR>;

multiclass vgathermhq<RegisterClass RC1, RegisterClass RC2> {
let isCodeGenOnly = 1, isPseudo = 1, mayLoad = 1,
mayStore = 1, addrMode = BaseImmOffset, accessSize = HalfWordAccess in
Expand Down
59 changes: 59 additions & 0 deletions llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ using namespace llvm;
static cl::opt<bool> HexagonAutoHVX("hexagon-autohvx", cl::init(false),
cl::Hidden, cl::desc("Enable loop vectorizer for HVX"));

cl::opt<bool> HexagonAllowScatterGatherHVX(
"hexagon-allow-scatter-gather-hvx", cl::init(false), cl::Hidden,
cl::desc("Allow auto-generation of HVX scatter-gather"));

static cl::opt<bool> EnableV68FloatAutoHVX(
"force-hvx-float", cl::Hidden,
cl::desc("Enable auto-vectorization of floatint point types on v68."));
Expand Down Expand Up @@ -354,6 +358,61 @@ bool HexagonTTIImpl::isLegalMaskedLoad(Type *DataType, Align /*Alignment*/,
return HexagonMaskedVMem && ST.isTypeForHVX(DataType);
}

bool HexagonTTIImpl::isLegalMaskedGather(Type *Ty, Align Alignment) const {
// For now assume we can not deal with all HVX datatypes.
if (!Ty->isVectorTy() || !ST.isTypeForHVX(Ty) ||
!HexagonAllowScatterGatherHVX)
return false;
// This must be in sync with HexagonVectorCombine pass.
switch (Ty->getScalarSizeInBits()) {
case 8:
return (getTypeNumElements(Ty) == 128);
case 16:
if (getTypeNumElements(Ty) == 64 || getTypeNumElements(Ty) == 32)
return (Alignment >= 2);
break;
case 32:
if (getTypeNumElements(Ty) == 32)
return (Alignment >= 4);
break;
default:
break;
}
return false;
}

bool HexagonTTIImpl::isLegalMaskedScatter(Type *Ty, Align Alignment) const {
if (!Ty->isVectorTy() || !ST.isTypeForHVX(Ty) ||
!HexagonAllowScatterGatherHVX)
return false;
// This must be in sync with HexagonVectorCombine pass.
switch (Ty->getScalarSizeInBits()) {
case 8:
return (getTypeNumElements(Ty) == 128);
case 16:
if (getTypeNumElements(Ty) == 64)
return (Alignment >= 2);
break;
case 32:
if (getTypeNumElements(Ty) == 32)
return (Alignment >= 4);
break;
default:
break;
}
return false;
}

bool HexagonTTIImpl::forceScalarizeMaskedGather(VectorType *VTy,
Align Alignment) const {
return !isLegalMaskedGather(VTy, Alignment);
}

bool HexagonTTIImpl::forceScalarizeMaskedScatter(VectorType *VTy,
Align Alignment) const {
return !isLegalMaskedScatter(VTy, Alignment);
}

/// --- Vector TTI end ---

unsigned HexagonTTIImpl::getPrefetchDistance() const {
Expand Down
6 changes: 6 additions & 0 deletions llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,12 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
unsigned AddressSpace) const override;
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
unsigned AddressSpace) const override;
bool isLegalMaskedGather(Type *Ty, Align Alignment) const override;
bool isLegalMaskedScatter(Type *Ty, Align Alignment) const override;
bool forceScalarizeMaskedGather(VectorType *VTy,
Align Alignment) const override;
bool forceScalarizeMaskedScatter(VectorType *VTy,
Align Alignment) const override;

/// @}

Expand Down
Loading