Skip to content

Commit

Permalink
[GlobalIsel] Combine G_EXTRACT_VECTOR_ELT (#85321)
Browse files Browse the repository at this point in the history 
preliminary steps
  • Loading branch information
@tschuett
tschuett committed Apr 2, 2024
1 parent 93c387d commit 8bb9443
Show file tree
Hide file tree
Showing 9 changed files with 930 additions and 25 deletions.
25 changes: 25 additions & 0 deletions llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,10 @@ class CombinerHelper {
/// This variant does not erase \p MI after calling the build function.
void applyBuildFnNoErase(MachineInstr &MI, BuildFnTy &MatchInfo);

/// Use a function which takes in a MachineIRBuilder to perform a combine.
/// By default, it erases the instruction \p MI from the function.
void applyBuildFnMO(const MachineOperand &MO, BuildFnTy &MatchInfo);

bool matchOrShiftToFunnelShift(MachineInstr &MI, BuildFnTy &MatchInfo);
bool matchFunnelShiftToRotate(MachineInstr &MI);
void applyFunnelShiftToRotate(MachineInstr &MI);
Expand Down Expand Up @@ -823,6 +827,27 @@ class CombinerHelper {
/// Combine addos.
bool matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo);

/// Combine extract vector element.
bool matchExtractVectorElement(MachineInstr &MI, BuildFnTy &MatchInfo);

/// Combine extract vector element with freeze on the vector register.
bool matchExtractVectorElementWithFreeze(const MachineOperand &MO,
BuildFnTy &MatchInfo);

/// Combine extract vector element with a build vector on the vector register.
bool matchExtractVectorElementWithBuildVector(const MachineOperand &MO,
BuildFnTy &MatchInfo);

/// Combine extract vector element with a build vector trunc on the vector
/// register.
bool matchExtractVectorElementWithBuildVectorTrunc(const MachineOperand &MO,
BuildFnTy &MatchInfo);

/// Combine extract vector element with a insert vector element on the vector
/// register and different indices.
bool matchExtractVectorElementWithDifferentIndices(const MachineOperand &MO,
BuildFnTy &MatchInfo);

private:
/// Checks for legality of an indexed variant of \p LdSt.
bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
Expand Down
41 changes: 41 additions & 0 deletions llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,14 @@ class GBuildVector : public GMergeLikeInstr {
}
};

/// Represents a G_BUILD_VECTOR_TRUNC.
class GBuildVectorTrunc : public GMergeLikeInstr {
public:
static bool classof(const MachineInstr *MI) {
return MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC;
}
};

/// Represents a G_PTR_ADD.
class GPtrAdd : public GenericMachineInstr {
public:
Expand Down Expand Up @@ -739,6 +747,39 @@ class GOr : public GLogicalBinOp {
};
};

/// Represents an extract vector element.
class GExtractVectorElement : public GenericMachineInstr {
public:
Register getVectorReg() const { return getOperand(1).getReg(); }
Register getIndexReg() const { return getOperand(2).getReg(); }

static bool classof(const MachineInstr *MI) {
return MI->getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT;
}
};

/// Represents an insert vector element.
class GInsertVectorElement : public GenericMachineInstr {
public:
Register getVectorReg() const { return getOperand(1).getReg(); }
Register getElementReg() const { return getOperand(2).getReg(); }
Register getIndexReg() const { return getOperand(3).getReg(); }

static bool classof(const MachineInstr *MI) {
return MI->getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
}
};

/// Represents a freeze.
class GFreeze : public GenericMachineInstr {
public:
Register getSourceReg() const { return getOperand(1).getReg(); }

static bool classof(const MachineInstr *MI) {
return MI->getOpcode() == TargetOpcode::G_FREEZE;
}
};

} // namespace llvm

#endif // LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H
230 changes: 228 additions & 2 deletions llvm/include/llvm/Target/GlobalISel/Combine.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1305,6 +1305,200 @@ def match_addos : GICombineRule<
[{ return Helper.matchAddOverflow(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;

def match_extract_of_element_undef_vector: GICombineRule <
(defs root:$root),
(match (G_IMPLICIT_DEF $vector),
(G_EXTRACT_VECTOR_ELT $root, $vector, $idx)),
(apply (G_IMPLICIT_DEF $root))
>;

def match_extract_of_element_undef_index: GICombineRule <
(defs root:$root),
(match (G_IMPLICIT_DEF $idx),
(G_EXTRACT_VECTOR_ELT $root, $vector, $idx)),
(apply (G_IMPLICIT_DEF $root))
>;

def match_extract_of_element : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root,
[{ return Helper.matchExtractVectorElement(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;

def extract_vector_element_not_const : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_INSERT_VECTOR_ELT $src, $x, $value, $idx),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx)),
(apply (GIReplaceReg $root, $value))>;

def extract_vector_element_different_indices : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_INSERT_VECTOR_ELT $src, $x, $value, $idx2),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx1),
[{ return Helper.matchExtractVectorElementWithDifferentIndices(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector2 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector3 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector4 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector5 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector6 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector7 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector8 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector9 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector10 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector11 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector12 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector13 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector14 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector15 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector16 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector_trunc2 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR_TRUNC $src, $x, $y),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector_trunc3 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector_trunc4 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector_trunc5 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector_trunc6 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b, $c),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector_trunc7 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b, $c, $d),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_build_vector_trunc8 : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_BUILD_VECTOR_TRUNC $src, $x, $y, $z, $a, $b, $c, $d, $e),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithBuildVectorTrunc(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

def extract_vector_element_freeze : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_FREEZE $src, $input),
(G_EXTRACT_VECTOR_ELT $root, $src, $idx),
[{ return Helper.matchExtractVectorElementWithFreeze(${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;

// Combines concat operations
def concat_matchinfo : GIDefMatchData<"SmallVector<Register>">;
def combine_concat_vector : GICombineRule<
Expand All @@ -1313,6 +1507,37 @@ def combine_concat_vector : GICombineRule<
[{ return Helper.matchCombineConcatVectors(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyCombineConcatVectors(*${root}, ${matchinfo}); }])>;

// match_extract_of_element must be the first!
def vector_ops_combines: GICombineGroup<[
match_extract_of_element_undef_vector,
match_extract_of_element_undef_index,
match_extract_of_element,
extract_vector_element_not_const,
extract_vector_element_different_indices,
extract_vector_element_build_vector2,
extract_vector_element_build_vector3,
extract_vector_element_build_vector4,
extract_vector_element_build_vector5,
extract_vector_element_build_vector7,
extract_vector_element_build_vector8,
extract_vector_element_build_vector9,
extract_vector_element_build_vector10,
extract_vector_element_build_vector11,
extract_vector_element_build_vector12,
extract_vector_element_build_vector13,
extract_vector_element_build_vector14,
extract_vector_element_build_vector15,
extract_vector_element_build_vector16,
extract_vector_element_build_vector_trunc2,
extract_vector_element_build_vector_trunc3,
extract_vector_element_build_vector_trunc4,
extract_vector_element_build_vector_trunc5,
extract_vector_element_build_vector_trunc6,
extract_vector_element_build_vector_trunc7,
extract_vector_element_build_vector_trunc8,
extract_vector_element_freeze
]>;

// FIXME: These should use the custom predicate feature once it lands.
def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
undef_to_negative_one,
Expand Down Expand Up @@ -1368,8 +1593,9 @@ def fma_combines : GICombineGroup<[combine_fadd_fmul_to_fmad_or_fma,
def constant_fold_binops : GICombineGroup<[constant_fold_binop,
constant_fold_fp_binop]>;

def all_combines : GICombineGroup<[trivial_combines, insert_vec_elt_combines,
extract_vec_elt_combines, combines_for_extload, combine_extracted_vector_load,
def all_combines : GICombineGroup<[trivial_combines, vector_ops_combines,
insert_vec_elt_combines, extract_vec_elt_combines, combines_for_extload,
combine_extracted_vector_load,
undef_combines, identity_combines, phi_combines,
simplify_add_to_sub, hoist_logic_op_with_same_opcode_hands, shifts_too_big,
reassocs, ptr_add_immed_chain,
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/CodeGen/GlobalISel/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ add_llvm_component_library(LLVMGlobalISel
GlobalISel.cpp
Combiner.cpp
CombinerHelper.cpp
CombinerHelperVectorOps.cpp
GIMatchTableExecutor.cpp
GISelChangeObserver.cpp
IRTranslator.cpp
Expand Down
8 changes: 8 additions & 0 deletions llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4058,6 +4058,14 @@ void CombinerHelper::applyBuildFn(
MI.eraseFromParent();
}

void CombinerHelper::applyBuildFnMO(const MachineOperand &MO,
BuildFnTy &MatchInfo) {
MachineInstr *Root = getDefIgnoringCopies(MO.getReg(), MRI);
Builder.setInstrAndDebugLoc(*Root);
MatchInfo(Builder);
Root->eraseFromParent();
}

void CombinerHelper::applyBuildFnNoErase(
MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) {
MatchInfo(Builder);
Expand Down

0 comments on commit 8bb9443

Please sign in to comment.