Skip to content

Commit

Permalink
[X86] Support promoted ENQCMD, KEYLOCKER and USERMSR (#77293)
Browse files Browse the repository at this point in the history 
R16-R31 was added into GPRs in
#70958,
This patch supports the promoted ENQCMD, KEYLOCKER and USER-MSR
instructions in EVEX space.

RFC:
https://discourse.llvm.org/t/rfc-design-for-apx-feature-egpr-and-ndd-support/73031/4
  • Loading branch information
@XinWang10
XinWang10 committed Jan 26, 2024
1 parent df5e431 commit 6d0080b
Show file tree
Hide file tree
Showing 20 changed files with 883 additions and 94 deletions.
2 changes: 2 additions & 0 deletions llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,9 @@ enum attributeBits {
ENUM_ENTRY(IC_EVEX, 1, "requires an EVEX prefix") \
ENUM_ENTRY(IC_EVEX_NF, 2, "requires EVEX and NF prefix") \
ENUM_ENTRY(IC_EVEX_XS, 2, "requires EVEX and the XS prefix") \
ENUM_ENTRY(IC_EVEX_XS_ADSIZE, 3, "requires EVEX, XS and the ADSIZE prefix") \
ENUM_ENTRY(IC_EVEX_XD, 2, "requires EVEX and the XD prefix") \
ENUM_ENTRY(IC_EVEX_XD_ADSIZE, 3, "requires EVEX, XD and the ADSIZE prefix") \
ENUM_ENTRY(IC_EVEX_OPSIZE, 2, "requires EVEX and the OpSize prefix") \
ENUM_ENTRY(IC_EVEX_OPSIZE_NF, 3, "requires EVEX, NF and the OpSize prefix") \
ENUM_ENTRY(IC_EVEX_OPSIZE_ADSIZE, 3, \
Expand Down
3 changes: 3 additions & 0 deletions llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -941,6 +941,9 @@ static bool readOpcode(struct InternalInstruction *insn) {
case VEX_LOB_MAP6:
insn->opcodeType = MAP6;
return consume(insn, insn->opcode);
case VEX_LOB_MAP7:
insn->opcodeType = MAP7;
return consume(insn, insn->opcode);
}
} else if (insn->vectorExtensionType == TYPE_VEX_3B) {
switch (mmmmmFromVEX2of3(insn->vectorExtensionPrefix[1])) {
Expand Down
20 changes: 12 additions & 8 deletions llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5035,14 +5035,18 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
unsigned Opcode;
switch (IntNo) {
default: llvm_unreachable("Impossible intrinsic");
case Intrinsic::x86_encodekey128: Opcode = X86::ENCODEKEY128; break;
case Intrinsic::x86_encodekey256: Opcode = X86::ENCODEKEY256; break;
case Intrinsic::x86_encodekey128:
Opcode = GET_EGPR_IF_ENABLED(X86::ENCODEKEY128);
break;
case Intrinsic::x86_encodekey256:
Opcode = GET_EGPR_IF_ENABLED(X86::ENCODEKEY256);
break;
}

SDValue Chain = Node->getOperand(0);
Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0, Node->getOperand(3),
SDValue());
if (Opcode == X86::ENCODEKEY256)
if (Opcode == X86::ENCODEKEY256 || Opcode == X86::ENCODEKEY256_EVEX)
Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1, Node->getOperand(4),
Chain.getValue(1));

Expand Down Expand Up @@ -5514,7 +5518,6 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
LoReg = UseMULX ? X86::RDX : X86::RAX;
HiReg = X86::RDX;
break;
#undef GET_EGPR_IF_ENABLED
}

SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
Expand Down Expand Up @@ -6394,17 +6397,18 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
default:
llvm_unreachable("Unexpected opcode!");
case X86ISD::AESENCWIDE128KL:
Opcode = X86::AESENCWIDE128KL;
Opcode = GET_EGPR_IF_ENABLED(X86::AESENCWIDE128KL);
break;
case X86ISD::AESDECWIDE128KL:
Opcode = X86::AESDECWIDE128KL;
Opcode = GET_EGPR_IF_ENABLED(X86::AESDECWIDE128KL);
break;
case X86ISD::AESENCWIDE256KL:
Opcode = X86::AESENCWIDE256KL;
Opcode = GET_EGPR_IF_ENABLED(X86::AESENCWIDE256KL);
break;
case X86ISD::AESDECWIDE256KL:
Opcode = X86::AESDECWIDE256KL;
Opcode = GET_EGPR_IF_ENABLED(X86::AESDECWIDE256KL);
break;
#undef GET_EGPR_IF_ENABLED
}

SDValue Chain = Node->getOperand(0);
Expand Down
100 changes: 57 additions & 43 deletions llvm/lib/Target/X86/X86InstrKL.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,61 +14,75 @@

//===----------------------------------------------------------------------===//
// Key Locker instructions
class Encodekey<bits<8> opcode, string m>
: I<opcode, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), m#"\t{$src, $dst|$dst, $src}", []>,
NoCD8, XS;

let SchedRW = [WriteSystem], Predicates = [HasKL] in {
let Uses = [XMM0, EAX], Defs = [EFLAGS] in {
multiclass Aesencdec<string suffix> {
def AESENC128KL#suffix : I<0xDC, MRMSrcMem, (outs VR128:$dst),
(ins VR128:$src1, opaquemem:$src2),
"aesenc128kl\t{$src2, $src1|$src1, $src2}",
[(set VR128:$dst, EFLAGS, (X86aesenc128kl VR128:$src1, addr:$src2))]>,
NoCD8, XS;
def AESDEC128KL#suffix : I<0xDD, MRMSrcMem, (outs VR128:$dst),
(ins VR128:$src1, opaquemem:$src2),
"aesdec128kl\t{$src2, $src1|$src1, $src2}",
[(set VR128:$dst, EFLAGS, (X86aesdec128kl VR128:$src1, addr:$src2))]>,
NoCD8, XS;
def AESENC256KL#suffix : I<0xDE, MRMSrcMem, (outs VR128:$dst),
(ins VR128:$src1, opaquemem:$src2),
"aesenc256kl\t{$src2, $src1|$src1, $src2}",
[(set VR128:$dst, EFLAGS, (X86aesenc256kl VR128:$src1, addr:$src2))]>,
NoCD8, XS;
def AESDEC256KL#suffix : I<0xDF, MRMSrcMem, (outs VR128:$dst),
(ins VR128:$src1, opaquemem:$src2),
"aesdec256kl\t{$src2, $src1|$src1, $src2}",
[(set VR128:$dst, EFLAGS, (X86aesdec256kl VR128:$src1, addr:$src2))]>,
NoCD8, XS;
}

let SchedRW = [WriteSystem] in {
let Uses = [XMM0, EAX], Defs = [EFLAGS], Predicates = [HasKL] in {
def LOADIWKEY : I<0xDC, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2),
"loadiwkey\t{$src2, $src1|$src1, $src2}",
[(int_x86_loadiwkey XMM0, VR128:$src1, VR128:$src2, EAX)]>, T8, XS;
}

let Uses = [XMM0], Defs = [XMM0, XMM1, XMM2, XMM4, XMM5, XMM6, EFLAGS] in {
def ENCODEKEY128 : I<0xFA, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src),
"encodekey128\t{$src, $dst|$dst, $src}", []>, T8, XS;
}
let Predicates = [HasKL, NoEGPR] in {
let Uses = [XMM0], Defs = [XMM0, XMM1, XMM2, XMM4, XMM5, XMM6, EFLAGS] in
def ENCODEKEY128 : Encodekey<0xFA, "encodekey128">, T8;

let Uses = [XMM0, XMM1], Defs = [XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, EFLAGS] in {
def ENCODEKEY256 : I<0xFB, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src),
"encodekey256\t{$src, $dst|$dst, $src}", []>, T8, XS;
}
let Uses = [XMM0, XMM1], Defs = [XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, EFLAGS] in
def ENCODEKEY256 : Encodekey<0xFB, "encodekey256">, T8;

let Constraints = "$src1 = $dst",
Defs = [EFLAGS] in {
def AESENC128KL : I<0xDC, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, opaquemem:$src2),
"aesenc128kl\t{$src2, $src1|$src1, $src2}",
[(set VR128:$dst, EFLAGS,
(X86aesenc128kl VR128:$src1, addr:$src2))]>, T8, XS;
let Constraints = "$src1 = $dst", Defs = [EFLAGS] in
defm "" : Aesencdec<"">, T8;
}

def AESDEC128KL : I<0xDD, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, opaquemem:$src2),
"aesdec128kl\t{$src2, $src1|$src1, $src2}",
[(set VR128:$dst, EFLAGS,
(X86aesdec128kl VR128:$src1, addr:$src2))]>, T8, XS;
let Predicates = [HasKL, HasEGPR, In64BitMode] in {
let Uses = [XMM0], Defs = [XMM0, XMM1, XMM2, XMM4, XMM5, XMM6, EFLAGS] in
def ENCODEKEY128_EVEX : Encodekey<0xDA, "encodekey128">, EVEX, T_MAP4;

def AESENC256KL : I<0xDE, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, opaquemem:$src2),
"aesenc256kl\t{$src2, $src1|$src1, $src2}",
[(set VR128:$dst, EFLAGS,
(X86aesenc256kl VR128:$src1, addr:$src2))]>, T8, XS;
let Uses = [XMM0, XMM1], Defs = [XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, EFLAGS] in
def ENCODEKEY256_EVEX : Encodekey<0xDB, "encodekey256">, EVEX, T_MAP4;

def AESDEC256KL : I<0xDF, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, opaquemem:$src2),
"aesdec256kl\t{$src2, $src1|$src1, $src2}",
[(set VR128:$dst, EFLAGS,
(X86aesdec256kl VR128:$src1, addr:$src2))]>, T8, XS;
let Constraints = "$src1 = $dst", Defs = [EFLAGS] in
defm "" : Aesencdec<"_EVEX">, EVEX, T_MAP4;
}
} // SchedRW

} // SchedRW, Predicates
multiclass Aesencdecwide<string suffix> {
def AESENCWIDE128KL#suffix : I<0xD8, MRM0m, (outs), (ins opaquemem:$src), "aesencwide128kl\t$src", []>, NoCD8, XS;
def AESDECWIDE128KL#suffix : I<0xD8, MRM1m, (outs), (ins opaquemem:$src), "aesdecwide128kl\t$src", []>, NoCD8, XS;
def AESENCWIDE256KL#suffix : I<0xD8, MRM2m, (outs), (ins opaquemem:$src), "aesencwide256kl\t$src", []>, NoCD8, XS;
def AESDECWIDE256KL#suffix : I<0xD8, MRM3m, (outs), (ins opaquemem:$src), "aesdecwide256kl\t$src", []>, NoCD8, XS;
}

let SchedRW = [WriteSystem], Predicates = [HasWIDEKL] in {
let Uses = [XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7],
Defs = [EFLAGS, XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7],
mayLoad = 1 in {
def AESENCWIDE128KL : I<0xD8, MRM0m, (outs), (ins opaquemem:$src),
"aesencwide128kl\t$src", []>, T8, XS;
def AESDECWIDE128KL : I<0xD8, MRM1m, (outs), (ins opaquemem:$src),
"aesdecwide128kl\t$src", []>, T8, XS;
def AESENCWIDE256KL : I<0xD8, MRM2m, (outs), (ins opaquemem:$src),
"aesencwide256kl\t$src", []>, T8, XS;
def AESDECWIDE256KL : I<0xD8, MRM3m, (outs), (ins opaquemem:$src),
"aesdecwide256kl\t$src", []>, T8, XS;
}
let SchedRW = [WriteSystem], Uses = [XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7],
Defs = [EFLAGS, XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7], mayLoad = 1 in {
let Predicates = [HasWIDEKL, NoEGPR] in
defm "" : Aesencdecwide<"">, T8;

} // SchedRW, Predicates
let Predicates = [HasWIDEKL, HasEGPR, In64BitMode] in
defm "" : Aesencdecwide<"_EVEX">, EVEX, T_MAP4;
} // SchedRW
51 changes: 29 additions & 22 deletions llvm/lib/Target/X86/X86InstrMisc.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1557,32 +1557,39 @@ def MOVDIR64B64_EVEX : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$
//===----------------------------------------------------------------------===//
// ENQCMD/S - Enqueue 64-byte command as user with 64-byte write atomicity
//
multiclass Enqcmds<string suffix> {
def ENQCMD32#suffix : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem_GR32:$src),
"enqcmd\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmd GR32:$dst, addr:$src))]>,
NoCD8, XD, AdSize32;
def ENQCMD64#suffix : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$src),
"enqcmd\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmd GR64:$dst, addr:$src))]>,
NoCD8, XD, AdSize64;

def ENQCMDS32#suffix : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem_GR32:$src),
"enqcmds\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmds GR32:$dst, addr:$src))]>,
NoCD8, XS, AdSize32;
def ENQCMDS64#suffix : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$src),
"enqcmds\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmds GR64:$dst, addr:$src))]>,
NoCD8, XS, AdSize64;
}

let SchedRW = [WriteStore], Defs = [EFLAGS] in {
def ENQCMD16 : I<0xF8, MRMSrcMem, (outs), (ins GR16:$dst, i512mem_GR16:$src),
"enqcmd\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmd GR16:$dst, addr:$src))]>,
"enqcmd\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmd GR16:$dst, addr:$src))]>,
T8, XD, AdSize16, Requires<[HasENQCMD, Not64BitMode]>;
def ENQCMD32 : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem_GR32:$src),
"enqcmd\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmd GR32:$dst, addr:$src))]>,
T8, XD, AdSize32, Requires<[HasENQCMD]>;
def ENQCMD64 : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$src),
"enqcmd\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmd GR64:$dst, addr:$src))]>,
T8, XD, AdSize64, Requires<[HasENQCMD, In64BitMode]>;

def ENQCMDS16 : I<0xF8, MRMSrcMem, (outs), (ins GR16:$dst, i512mem_GR16:$src),
"enqcmds\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmds GR16:$dst, addr:$src))]>,
T8, XS, AdSize16, Requires<[HasENQCMD, Not64BitMode]>;
def ENQCMDS32 : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem_GR32:$src),
"enqcmds\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmds GR32:$dst, addr:$src))]>,
T8, XS, AdSize32, Requires<[HasENQCMD]>;
def ENQCMDS64 : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$src),
"enqcmds\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmds GR64:$dst, addr:$src))]>,
T8, XS, AdSize64, Requires<[HasENQCMD, In64BitMode]>;
"enqcmds\t{$src, $dst|$dst, $src}",
[(set EFLAGS, (X86enqcmds GR16:$dst, addr:$src))]>,
T8, XS, AdSize16, Requires<[HasENQCMD, Not64BitMode]>;

defm "" : Enqcmds<"">, T8, Requires<[HasENQCMD, NoEGPR]>;
defm "" : Enqcmds<"_EVEX">, EVEX, T_MAP4, Requires<[HasENQCMD, HasEGPR, In64BitMode]>;

}

//===----------------------------------------------------------------------===//
Expand Down
41 changes: 27 additions & 14 deletions llvm/lib/Target/X86/X86InstrSystem.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -436,22 +436,35 @@ def WRMSRLIST : I<0x01, MRM_C6, (outs), (ins), "wrmsrlist", []>, TB, XS;
def RDMSRLIST : I<0x01, MRM_C6, (outs), (ins), "rdmsrlist", []>, TB, XD;
}

let Predicates = [HasUSERMSR], mayLoad = 1 in {
def URDMSRrr : I<0xf8, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src),
"urdmsr\t{$src, $dst|$dst, $src}",
[(set GR64:$dst, (int_x86_urdmsr GR64:$src))]>, T8, XD;
def URDMSRri : Ii32<0xf8, MRM0r, (outs GR64:$dst), (ins i64i32imm:$imm),
"urdmsr\t{$imm, $dst|$dst, $imm}",
[(set GR64:$dst, (int_x86_urdmsr i64immSExt32_su:$imm))]>, T_MAP7, XD, VEX;
multiclass Urdwrmsr<Map rrmap, string suffix> {
let mayLoad = 1 in {
let OpMap = rrmap in
def URDMSRrr#suffix : I<0xf8, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src),
"urdmsr\t{$src, $dst|$dst, $src}",
[(set GR64:$dst, (int_x86_urdmsr GR64:$src))]>, XD, NoCD8;
def URDMSRri#suffix : Ii32<0xf8, MRM0r, (outs GR64:$dst), (ins i64i32imm:$imm),
"urdmsr\t{$imm, $dst|$dst, $imm}",
[(set GR64:$dst, (int_x86_urdmsr i64immSExt32_su:$imm))]>,
T_MAP7, VEX, XD, NoCD8;
}
let Predicates = [HasUSERMSR], mayStore = 1 in {
def UWRMSRrr : I<0xf8, MRMSrcReg, (outs), (ins GR64:$src1, GR64:$src2),
"uwrmsr\t{$src2, $src1|$src1, $src2}",
[(int_x86_uwrmsr GR64:$src1, GR64:$src2)]>, T8, XS;
def UWRMSRir : Ii32<0xf8, MRM0r, (outs), (ins GR64:$src, i64i32imm:$imm),
"uwrmsr\t{$src, $imm|$imm, $src}",
[(int_x86_uwrmsr i64immSExt32_su:$imm, GR64:$src)]>, T_MAP7, XS, VEX;
let mayStore = 1 in {
let OpMap = rrmap in
def UWRMSRrr#suffix : I<0xf8, MRMSrcReg, (outs), (ins GR64:$src1, GR64:$src2),
"uwrmsr\t{$src2, $src1|$src1, $src2}",
[(int_x86_uwrmsr GR64:$src1, GR64:$src2)]>, XS, NoCD8;
def UWRMSRir#suffix : Ii32<0xf8, MRM0r, (outs), (ins GR64:$src, i64i32imm:$imm),
"uwrmsr\t{$src, $imm|$imm, $src}",
[(int_x86_uwrmsr i64immSExt32_su:$imm, GR64:$src)]>,
T_MAP7, VEX, XS, NoCD8;
}
}

let Predicates = [HasUSERMSR, NoEGPR] in
defm "" : Urdwrmsr<T8, "">;

let Predicates = [HasUSERMSR, HasEGPR, In64BitMode] in
defm "" : Urdwrmsr<T_MAP4, "_EVEX">, EVEX;

let Defs = [RAX, RDX], Uses = [ECX] in
def RDPMC : I<0x33, RawFrm, (outs), (ins), "rdpmc", []>, TB;

Expand Down
13 changes: 13 additions & 0 deletions llvm/test/CodeGen/X86/enqcmd-intrinsics.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+enqcmd | FileCheck %s --check-prefix=X64
; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+enqcmd | FileCheck %s --check-prefix=X86
; RUN: llc < %s -mtriple=x86_64-linux-gnux32 -mattr=+enqcmd | FileCheck %s --check-prefix=X32
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+enqcmd,+egpr --show-mc-encoding | FileCheck %s --check-prefix=EGPR

define i8 @test_enqcmd(ptr %dst, ptr %src) {
; X64-LABEL: test_enqcmd:
Expand All @@ -23,6 +24,12 @@ define i8 @test_enqcmd(ptr %dst, ptr %src) {
; X32-NEXT: enqcmd (%esi), %edi
; X32-NEXT: sete %al
; X32-NEXT: retq
;
; EGPR-LABEL: test_enqcmd:
; EGPR: # %bb.0: # %entry
; EGPR-NEXT: enqcmd (%rsi), %rdi # EVEX TO LEGACY Compression encoding: [0xf2,0x0f,0x38,0xf8,0x3e]
; EGPR-NEXT: sete %al # encoding: [0x0f,0x94,0xc0]
; EGPR-NEXT: retq # encoding: [0xc3]
entry:


Expand Down Expand Up @@ -50,6 +57,12 @@ define i8 @test_enqcmds(ptr %dst, ptr %src) {
; X32-NEXT: enqcmds (%esi), %edi
; X32-NEXT: sete %al
; X32-NEXT: retq
;
; EGPR-LABEL: test_enqcmds:
; EGPR: # %bb.0: # %entry
; EGPR-NEXT: enqcmds (%rsi), %rdi # EVEX TO LEGACY Compression encoding: [0xf3,0x0f,0x38,0xf8,0x3e]
; EGPR-NEXT: sete %al # encoding: [0x0f,0x94,0xc0]
; EGPR-NEXT: retq # encoding: [0xc3]
entry:


Expand Down

0 comments on commit 6d0080b

Please sign in to comment.