Skip to content

[AMDGPU][MC] Do not inline lit()/lit64() operands. #162137

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 8, 2025
+417 −78
Merged

[AMDGPU][MC] Do not inline lit()/lit64() operands. #162137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 34 additions & 51 deletions llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2105,6 +2105,10 @@ bool AMDGPUOperand::isInlinableImm(MVT type) const {
// Only plain immediates are inlinable (e.g. "clamp" attribute is not)
return false;
}

if (getModifiers().Lit != LitModifier::None)
return false;

// TODO: We should avoid using host float here. It would be better to
// check the float bit values which is what a few other places do.
// We've had bot failures before due to weird NaN support on mips hosts.
Expand Down Expand Up @@ -2349,7 +2353,8 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
case AMDGPU::OPERAND_REG_INLINE_C_INT64:
case AMDGPU::OPERAND_REG_INLINE_C_FP64:
case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
if (Lit == LitModifier::None &&
AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
AsmParser->hasInv2PiInlineImm())) {
Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
return;
Expand Down Expand Up @@ -2386,14 +2391,7 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
Val = Hi_32(Val);
}
}

if (Lit != LitModifier::None) {
Inst.addOperand(
MCOperand::createExpr(AMDGPUMCExpr::createLit(Lit, Val, Ctx)));
} else {
Inst.addOperand(MCOperand::createImm(Val));
}
return;
break;
}

// We don't allow fp literals in 64-bit integer instructions. It is
Expand All @@ -2405,20 +2403,14 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
if (CanUse64BitLiterals && Lit == LitModifier::None &&
(isInt<32>(Val) || isUInt<32>(Val)))
Lit = LitModifier::Lit64;

if (Lit != LitModifier::None) {
Inst.addOperand(
MCOperand::createExpr(AMDGPUMCExpr::createLit(Lit, Val, Ctx)));
} else {
Inst.addOperand(MCOperand::createImm(Val));
}
return;
break;

case AMDGPU::OPERAND_REG_IMM_BF16:
case AMDGPU::OPERAND_REG_INLINE_C_BF16:
case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
case AMDGPU::OPERAND_REG_IMM_V2BF16:
if (AsmParser->hasInv2PiInlineImm() && Literal == 0x3fc45f306725feed) {
if (Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
Literal == 0x3fc45f306725feed) {
// This is the 1/(2*pi) which is going to be truncated to bf16 with the
// loss of precision. The constant represents ideomatic fp32 value of
// 1/(2*pi) = 0.15915494 since bf16 is in fact fp32 with cleared low 16
Expand Down Expand Up @@ -2456,14 +2448,19 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
// We allow precision lost but not overflow or underflow. This should be
// checked earlier in isLiteralImm()

uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
Inst.addOperand(MCOperand::createImm(ImmVal));
return;
Val = FPLiteral.bitcastToAPInt().getZExtValue();
break;
}
default:
llvm_unreachable("invalid operand size");
}

if (Lit != LitModifier::None) {
Inst.addOperand(
MCOperand::createExpr(AMDGPUMCExpr::createLit(Lit, Val, Ctx)));
} else {
Inst.addOperand(MCOperand::createImm(Val));
}
return;
}

Expand All @@ -2483,12 +2480,12 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
case AMDGPU::OPERAND_REG_IMM_V2INT32:
case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
Inst.addOperand(MCOperand::createImm(Val));
return;
break;

case AMDGPU::OPERAND_REG_IMM_INT64:
case AMDGPU::OPERAND_REG_INLINE_C_INT64:
if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
if (Lit == LitModifier::None &&
AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
Inst.addOperand(MCOperand::createImm(Val));
return;
}
Expand All @@ -2499,19 +2496,13 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
// LSBs.
if (!AsmParser->has64BitLiterals() || Lit == LitModifier::Lit)
Val = Lo_32(Val);

if (Lit != LitModifier::None) {
Inst.addOperand(
MCOperand::createExpr(AMDGPUMCExpr::createLit(Lit, Val, Ctx)));
} else {
Inst.addOperand(MCOperand::createImm(Val));
}
return;
break;

case AMDGPU::OPERAND_REG_IMM_FP64:
case AMDGPU::OPERAND_REG_INLINE_C_FP64:
case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
if (Lit == LitModifier::None &&
AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
Inst.addOperand(MCOperand::createImm(Val));
return;
}
Expand All @@ -2534,14 +2525,7 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
// For FP64 operands lit() specifies the high half of the value.
if (Lit == LitModifier::Lit)
Val = Hi_32(Val);

if (Lit != LitModifier::None) {
Inst.addOperand(
MCOperand::createExpr(AMDGPUMCExpr::createLit(Lit, Val, Ctx)));
} else {
Inst.addOperand(MCOperand::createImm(Val));
}
return;
break;

case AMDGPU::OPERAND_REG_IMM_INT16:
case AMDGPU::OPERAND_REG_INLINE_C_INT16:
Expand All @@ -2554,24 +2538,23 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
case AMDGPU::OPERAND_KIMM32:
case AMDGPU::OPERAND_KIMM16:
Inst.addOperand(MCOperand::createImm(Val));
return;
break;

case AMDGPU::OPERAND_KIMM64:
if ((isInt<32>(Val) || isUInt<32>(Val)) && Lit != LitModifier::Lit64)
Val <<= 32;

if (Lit != LitModifier::None) {
Inst.addOperand(
MCOperand::createExpr(AMDGPUMCExpr::createLit(Lit, Val, Ctx)));
} else {
Inst.addOperand(MCOperand::createImm(Val));
}
return;
break;

default:
llvm_unreachable("invalid operand type");
}

if (Lit != LitModifier::None) {
Inst.addOperand(
MCOperand::createExpr(AMDGPUMCExpr::createLit(Lit, Val, Ctx)));
} else {
Inst.addOperand(MCOperand::createImm(Val));
}
}

void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
Expand Down
8 changes: 7 additions & 1 deletion llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,13 @@ void AMDGPUInstPrinter::printU16ImmDecOperand(const MCInst *MI, unsigned OpNo,
void AMDGPUInstPrinter::printU32ImmOperand(const MCInst *MI, unsigned OpNo,
const MCSubtargetInfo &STI,
raw_ostream &O) {
O << formatHex(MI->getOperand(OpNo).getImm() & 0xffffffff);
const MCOperand &Op = MI->getOperand(OpNo);
if (Op.isExpr()) {
MAI.printExpr(O, *Op.getExpr());
return;
}

O << formatHex(Op.getImm() & 0xffffffff);
}

void AMDGPUInstPrinter::printFP64ImmOperand(const MCInst *MI, unsigned OpNo,
Expand Down
15 changes: 12 additions & 3 deletions llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,10 +270,19 @@ std::optional<uint64_t> AMDGPUMCCodeEmitter::getLitEncoding(
const MCInstrDesc &Desc, const MCOperand &MO, unsigned OpNo,
const MCSubtargetInfo &STI, bool HasMandatoryLiteral) const {
const MCOperandInfo &OpInfo = Desc.operands()[OpNo];
int64_t Imm;
int64_t Imm = 0;
if (MO.isExpr()) {
if (!MO.getExpr()->evaluateAsAbsolute(Imm))
return AMDGPU::getOperandSize(OpInfo) == 8 ? 254 : 255;
if (!MO.getExpr()->evaluateAsAbsolute(Imm) ||
AMDGPU::isLitExpr(MO.getExpr())) {
if (OpInfo.OperandType == AMDGPU::OPERAND_KIMM16 ||
OpInfo.OperandType == AMDGPU::OPERAND_KIMM32 ||
OpInfo.OperandType == AMDGPU::OPERAND_KIMM64)
return Imm;
if (STI.hasFeature(AMDGPU::Feature64BitLiterals) &&
AMDGPU::getOperandSize(OpInfo) == 8)
return 254;
return 255;
}
} else {
assert(!MO.isDFPImm());

Expand Down
3 changes: 2 additions & 1 deletion llvm/test/MC/AMDGPU/gfx1250_asm_salu_lit64.s
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,9 @@ s_cselect_b64 s[2:3], s[4:5], 0x10abcdef12345678
s_mov_b64 s[2:3], 0xffffffff01234567
// GFX1250: s_mov_b64 s[2:3], 0xffffffff01234567 ; encoding: [0xfe,0x01,0x82,0xbe,0x67,0x45,0x23,0x01,0xff,0xff,0xff,0xff]

// TODO: disasm
s_mov_b64 s[2:3], lit64(0x777)
// GFX1250-ASM: s_mov_b64 s[2:3], lit64(0x777) ; encoding: [0xff,0x01,0x82,0xbe,0x77,0x07,0x00,0x00]
// GFX1250-ASM: s_mov_b64 s[2:3], lit64(0x777) ; encoding: [0xfe,0x01,0x82,0xbe,0x77,0x07,0x00,0x00,0x00,0x00,0x00,0x00]
// GFX1250-DIS: s_mov_b64 s[2:3], 0x777 ; encoding: [0xff,0x01,0x82,0xbe,0x77,0x07,0x00,0x00]

s_mov_b64 s[2:3], 0x777
Expand Down
Loading