Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

x64: Add more support for more AVX instructions #5931

Merged
merged 7 commits into from
Mar 10, 2023
Merged

x64: Add more support for more AVX instructions #5931

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1c3a1bd
x64: Add a smattering of lowerings for `shuffle` specializations (#5930)
alexcrichton Mar 9, 2023
e9763f7
x64: Add an AVX encoding for the `pshufd` instruction
alexcrichton Mar 3, 2023
4170e37
x64: Add more support for AVX loads/stores
alexcrichton Mar 3, 2023
e008ecb
x64: Enable using AVX instructions for zero regs
alexcrichton Mar 3, 2023
312eb40
x64: Remove `XmmConstOp` as an instruction
alexcrichton Mar 3, 2023
eea5333
Review comments
alexcrichton Mar 9, 2023
1fd6e36
Update test expectations
alexcrichton Mar 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 181 additions & 64 deletions cranelift/codegen/src/isa/x64/inst.isle
View file
Open in desktop

Large diffs are not rendered by default.

34 changes: 32 additions & 2 deletions cranelift/codegen/src/isa/x64/inst/args.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1117,6 +1117,12 @@ pub enum SseOpcode {
Xorpd,
Phaddw,
Phaddd,
Punpckhdq,
Punpckldq,
Punpckhqdq,
Punpcklqdq,
Pshuflw,
Pshufhw,
}

impl SseOpcode {
Expand Down Expand Up @@ -1256,7 +1262,13 @@ impl SseOpcode {
| SseOpcode::Subpd
| SseOpcode::Subsd
| SseOpcode::Ucomisd
| SseOpcode::Xorpd => SSE2,
| SseOpcode::Xorpd
| SseOpcode::Punpckldq
| SseOpcode::Punpckhdq
| SseOpcode::Punpcklqdq
| SseOpcode::Punpckhqdq
| SseOpcode::Pshuflw
| SseOpcode::Pshufhw => SSE2,

SseOpcode::Pabsb
| SseOpcode::Pabsw
Expand Down Expand Up @@ -1501,6 +1513,12 @@ impl fmt::Debug for SseOpcode {
SseOpcode::Xorpd => "xorpd",
SseOpcode::Phaddw => "phaddw",
SseOpcode::Phaddd => "phaddd",
SseOpcode::Punpckldq => "punpckldq",
SseOpcode::Punpckhdq => "punpckhdq",
SseOpcode::Punpcklqdq => "punpcklqdq",
SseOpcode::Punpckhqdq => "punpckhqdq",
SseOpcode::Pshuflw => "pshuflw",
SseOpcode::Pshufhw => "pshufhw",
};
write!(fmt, "{}", name)
}
Expand Down Expand Up @@ -1669,7 +1687,19 @@ impl AvxOpcode {
| AvxOpcode::Vcvttpd2dq
| AvxOpcode::Vcvttps2dq
| AvxOpcode::Vphaddw
| AvxOpcode::Vphaddd => {
| AvxOpcode::Vphaddd
| AvxOpcode::Vpunpckldq
| AvxOpcode::Vpunpckhdq
| AvxOpcode::Vpunpcklqdq
| AvxOpcode::Vpunpckhqdq
| AvxOpcode::Vpshuflw
| AvxOpcode::Vpshufhw
| AvxOpcode::Vpshufd
| AvxOpcode::Vmovss
| AvxOpcode::Vmovsd
| AvxOpcode::Vmovups
| AvxOpcode::Vmovupd
| AvxOpcode::Vmovdqu => {
smallvec![InstructionSet::AVX]
}
}
Expand Down
69 changes: 53 additions & 16 deletions cranelift/codegen/src/isa/x64/inst/emit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1789,6 +1789,8 @@ pub(crate) fn emit(
SseOpcode::Roundpd => (LegacyPrefixes::_66, 0x0F3A09, 3),
SseOpcode::Roundsd => (LegacyPrefixes::_66, 0x0F3A0B, 3),
SseOpcode::Pshufd => (LegacyPrefixes::_66, 0x0F70, 2),
SseOpcode::Pshuflw => (LegacyPrefixes::_F2, 0x0F70, 2),
SseOpcode::Pshufhw => (LegacyPrefixes::_F3, 0x0F70, 2),
_ => unimplemented!("Opcode {:?} not implemented", op),
};
match src {
Expand Down Expand Up @@ -1946,6 +1948,10 @@ pub(crate) fn emit(
SseOpcode::Punpckhwd => (LegacyPrefixes::_66, 0x0F69, 2),
SseOpcode::Punpcklbw => (LegacyPrefixes::_66, 0x0F60, 2),
SseOpcode::Punpcklwd => (LegacyPrefixes::_66, 0x0F61, 2),
SseOpcode::Punpckldq => (LegacyPrefixes::_66, 0x0F62, 2),
SseOpcode::Punpcklqdq => (LegacyPrefixes::_66, 0x0F6C, 2),
SseOpcode::Punpckhdq => (LegacyPrefixes::_66, 0x0F6A, 2),
SseOpcode::Punpckhqdq => (LegacyPrefixes::_66, 0x0F6D, 2),
SseOpcode::Pxor => (LegacyPrefixes::_66, 0x0FEF, 2),
SseOpcode::Subps => (LegacyPrefixes::None, 0x0F5C, 2),
SseOpcode::Subpd => (LegacyPrefixes::_66, 0x0F5C, 2),
Expand All @@ -1970,22 +1976,6 @@ pub(crate) fn emit(
}
}

Inst::XmmConstOp { op, dst } => {
let dst = allocs.next(dst.to_reg().to_reg());
emit(
&Inst::XmmRmR {
op: *op,
dst: Writable::from_reg(Xmm::new(dst).unwrap()),
src1: Xmm::new(dst).unwrap(),
src2: Xmm::new(dst).unwrap().into(),
},
allocs,
sink,
info,
state,
);
}

Inst::XmmRmRBlend {
op,
src1,
Expand Down Expand Up @@ -2171,6 +2161,11 @@ pub(crate) fn emit(
AvxOpcode::Vmaxsd => (LP::_F2, OM::_0F, 0x5F),
AvxOpcode::Vphaddw => (LP::_66, OM::_0F38, 0x01),
AvxOpcode::Vphaddd => (LP::_66, OM::_0F38, 0x02),
AvxOpcode::Vpunpckldq => (LP::_66, OM::_0F, 0x62),
AvxOpcode::Vpunpckhdq => (LP::_66, OM::_0F, 0x6A),
AvxOpcode::Vpunpcklqdq => (LP::_66, OM::_0F, 0x6C),
AvxOpcode::Vpunpckhqdq => (LP::_66, OM::_0F, 0x6D),
AvxOpcode::Vmovsd => (LP::_F2, OM::_0F, 0x10),
_ => panic!("unexpected rmir vex opcode {op:?}"),
};
VexInstruction::new()
Expand Down Expand Up @@ -2375,6 +2370,23 @@ pub(crate) fn emit(
AvxOpcode::Vcvtps2pd => (LegacyPrefixes::None, OpcodeMap::_0F, 0x5A),
AvxOpcode::Vcvttpd2dq => (LegacyPrefixes::_66, OpcodeMap::_0F, 0xE6),
AvxOpcode::Vcvttps2dq => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x5B),
AvxOpcode::Vmovdqu => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x6F),
AvxOpcode::Vmovups => (LegacyPrefixes::None, OpcodeMap::_0F, 0x10),
AvxOpcode::Vmovupd => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x10),

// Note that for `vmov{s,d}` the `inst.isle` rules should
// statically ensure that only `Amode` operands are used here.
// Otherwise the other encodings of `vmovss` are more like
// 2-operand instructions which this unary encoding does not
// have.
AvxOpcode::Vmovss => match &src {
RegisterOrAmode::Amode(_) => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x10),
_ => unreachable!(),
},
AvxOpcode::Vmovsd => match &src {
RegisterOrAmode::Amode(_) => (LegacyPrefixes::_F2, OpcodeMap::_0F, 0x10),
_ => unreachable!(),
},
_ => panic!("unexpected rmr_imm_vex opcode {op:?}"),
};

Expand All @@ -2400,6 +2412,9 @@ pub(crate) fn emit(
let (prefix, map, opcode) = match op {
AvxOpcode::Vroundps => (LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x08),
AvxOpcode::Vroundpd => (LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x09),
AvxOpcode::Vpshuflw => (LegacyPrefixes::_F2, OpcodeMap::_0F, 0x70),
AvxOpcode::Vpshufhw => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x70),
AvxOpcode::Vpshufd => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x70),
_ => panic!("unexpected rmr_imm_vex opcode {op:?}"),
};

Expand All @@ -2414,6 +2429,28 @@ pub(crate) fn emit(
.encode(sink);
}

Inst::XmmMovRMVex { op, src, dst } => {
let src = allocs.next(*src);
let dst = dst.with_allocs(allocs).finalize(state, sink);

let (prefix, map, opcode) = match op {
AvxOpcode::Vmovdqu => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x7F),
AvxOpcode::Vmovss => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x11),
AvxOpcode::Vmovsd => (LegacyPrefixes::_F2, OpcodeMap::_0F, 0x11),
AvxOpcode::Vmovups => (LegacyPrefixes::None, OpcodeMap::_0F, 0x11),
AvxOpcode::Vmovupd => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x11),
_ => unimplemented!("Opcode {:?} not implemented", op),
};
VexInstruction::new()
.length(VexVectorLength::V128)
.prefix(prefix)
.map(map)
.opcode(opcode)
.rm(dst)
.reg(src.to_real_reg().unwrap().hw_enc())
.encode(sink);
}

Inst::XmmRmREvex {
op,
src1,
Expand Down
22 changes: 10 additions & 12 deletions cranelift/codegen/src/isa/x64/inst/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,7 @@ impl Inst {
| Inst::XmmToGprImm { op, .. }
| Inst::XmmUnaryRmRImm { op, .. }
| Inst::XmmUnaryRmRUnaligned { op, .. }
| Inst::XmmUnaryRmR { op, .. }
| Inst::XmmConstOp { op, .. } => smallvec![op.available_from()],
| Inst::XmmUnaryRmR { op, .. } => smallvec![op.available_from()],

Inst::XmmUnaryRmREvex { op, .. }
| Inst::XmmRmREvex { op, .. }
Expand All @@ -153,7 +152,8 @@ impl Inst {
| Inst::XmmRmRBlendVex { op, .. }
| Inst::XmmVexPinsr { op, .. }
| Inst::XmmUnaryRmRVex { op, .. }
| Inst::XmmUnaryRmRImmVex { op, .. } => op.available_from(),
| Inst::XmmUnaryRmRImmVex { op, .. }
| Inst::XmmMovRMVex { op, .. } => op.available_from(),
}
}
}
Expand Down Expand Up @@ -938,6 +938,12 @@ impl PrettyPrint for Inst {
format!("{} {}, {}", ljustify(op.to_string()), src, dst)
}

Inst::XmmMovRMVex { op, src, dst, .. } => {
let src = pretty_print_reg(*src, 8, allocs);
let dst = dst.pretty_print(8, allocs);
format!("{} {}, {}", ljustify(op.to_string()), src, dst)
}

Inst::XmmRmR {
op,
src1,
Expand All @@ -964,11 +970,6 @@ impl PrettyPrint for Inst {
format!("{} {}, {}, {}", ljustify(op.to_string()), src1, src2, dst)
}

Inst::XmmConstOp { op, dst } => {
let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
format!("{} {dst}, {dst}, {dst}", ljustify(op.to_string()))
}

Inst::XmmRmRBlend {
op,
src1,
Expand Down Expand Up @@ -2019,9 +2020,6 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
collector.reg_reuse_def(*dst, 0);
src2.get_operands(collector);
}
Inst::XmmConstOp { dst, .. } => {
collector.reg_def(dst.to_writable_reg());
}
Inst::XmmUninitializedValue { dst } => collector.reg_def(dst.to_writable_reg()),
Inst::XmmMinMaxSeq { lhs, rhs, dst, .. } => {
collector.reg_use(rhs.to_reg());
Expand All @@ -2035,7 +2033,7 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
collector.reg_reuse_def(dst.to_writable_reg(), 0); // Reuse RHS.
src2.get_operands(collector);
}
Inst::XmmMovRM { src, dst, .. } => {
Inst::XmmMovRM { src, dst, .. } | Inst::XmmMovRMVex { src, dst, .. } => {
collector.reg_use(*src);
dst.get_operands(collector);
}
Expand Down
Loading