From 53725e83bad0230ab9498657eb466881903fd806 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Wed, 1 Oct 2025 12:26:15 +0200 Subject: [PATCH] [GlobalISel] Add support for value/constants as inline asm memory operand --- .../CodeGen/GlobalISel/MachineIRBuilder.h | 13 +++ .../CodeGen/GlobalISel/InlineAsmLowering.cpp | 59 +++++++++---- .../CodeGen/GlobalISel/LegalizerHelper.cpp | 4 +- .../CodeGen/GlobalISel/MachineIRBuilder.cpp | 8 ++ .../AArch64/GlobalISel/arm64-fallback.ll | 9 -- .../GlobalISel/irtranslator-inline-asm.ll | 85 ++++++++++++++++++ .../GlobalISel/irtranslator-inline-asm.ll | 86 +++++++++++++++++++ 7 files changed, 237 insertions(+), 27 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 0b6033b4ba60a..2dab265614531 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -502,6 +502,19 @@ class LLVM_ABI MachineIRBuilder { /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildConstantPool(const DstOp &Res, unsigned Idx); + /// Build and insert \p Res = G_CONSTANT_POOL \p Idx + /// + /// G_CONSTANT_POOL materializes the address of an object in the constant + /// pool. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res must be a generic virtual register with pointer type. + /// + /// \param Idx LLVM Constant that will be inserted into the constant pool. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildConstantPool(const DstOp &Res, const Constant *V); + /// Build and insert \p Res = G_PTR_ADD \p Op0, \p Op1 /// /// G_PTR_ADD adds \p Op1 addressible units to the pointer specified by \p Op0, diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp index b4e64d7416d86..0bc5b38ea3250 100644 --- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp @@ -13,6 +13,8 @@ #include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/MachineConstantPool.h" +#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetLowering.h" @@ -454,26 +456,53 @@ bool InlineAsmLowering::lowerInlineAsm( } if (OpInfo.ConstraintType == TargetLowering::C_Memory) { - - if (!OpInfo.isIndirect) { - LLVM_DEBUG(dbgs() - << "Cannot indirectify memory input operands yet\n"); - return false; - } - - assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!"); - const InlineAsm::ConstraintCode ConstraintID = TLI->getInlineAsmMemConstraint(OpInfo.ConstraintCode); InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1); OpFlags.setMemConstraint(ConstraintID); Inst.addImm(OpFlags); - ArrayRef SourceRegs = - GetOrCreateVRegs(*OpInfo.CallOperandVal); - assert( - SourceRegs.size() == 1 && - "Expected the memory input to fit into a single virtual register"); - Inst.addReg(SourceRegs[0]); + + if (OpInfo.isIndirect) { + // already indirect + ArrayRef SourceRegs = + GetOrCreateVRegs(*OpInfo.CallOperandVal); + assert(SourceRegs.size() == 1 && "Expected the memory input to fit " + "into a single virtual register"); + Inst.addReg(SourceRegs[0]); + break; + } + + // Needs to be made indirect. + Value *OpVal = OpInfo.CallOperandVal; + // Constants go into the constant pool. + if (isa(OpVal) || isa(OpVal) || + isa(OpVal) || isa(OpVal)) { + unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace(); + LLT AddrPtrTy = + LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace)); + auto Addr = + MIRBuilder.buildConstantPool(AddrPtrTy, cast(OpVal)); + Inst.addReg(Addr.getReg(0)); + } else { + // TODO: Is using the DL instead of OpInfo fine here? + unsigned Bytes = DL.getTypeStoreSize(OpVal->getType()); + Align Alignment = DL.getPrefTypeAlign(OpVal->getType()); + int FrameIdx = + MF.getFrameInfo().CreateStackObject(Bytes, Alignment, false); + + unsigned AddrSpace = DL.getAllocaAddrSpace(); + LLT FramePtrTy = + LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace)); + auto Ptr = MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx).getReg(0); + ArrayRef SourceRegs = + GetOrCreateVRegs(*OpInfo.CallOperandVal); + assert(SourceRegs.size() == 1 && "Expected the memory input to fit " + "into a single virtual register"); + MIRBuilder.buildStore(SourceRegs[0], Ptr, + MachinePointerInfo::getFixedStack(MF, FrameIdx), + Alignment); + Inst.addReg(Ptr); + } break; } diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 03dfa6f3f243f..3a52f3c04579c 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -3519,9 +3519,7 @@ static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal, Align Alignment(DL.getABITypeAlign(ConstVal->getType())); - auto Addr = MIRBuilder.buildConstantPool( - AddrPtrTy, - MF.getConstantPool()->getConstantPoolIndex(ConstVal, Alignment)); + auto Addr = MIRBuilder.buildConstantPool(AddrPtrTy, ConstVal); MachineMemOperand *MMO = MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF), diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 27df7e369436a..0bff391b58363 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -9,6 +9,7 @@ /// This file implements the MachineIRBuidler class. //===----------------------------------------------------------------------===// #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" @@ -175,6 +176,13 @@ MachineInstrBuilder MachineIRBuilder::buildConstantPool(const DstOp &Res, return MIB; } +MachineInstrBuilder MachineIRBuilder::buildConstantPool(const DstOp &Res, + const Constant *V) { + Align Alignment(getDataLayout().getABITypeAlign(V->getType())); + return buildConstantPool( + Res, getMF().getConstantPool()->getConstantPoolIndex(V, Alignment)); +} + MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy, unsigned JTI) { return buildInstr(TargetOpcode::G_JUMP_TABLE, {PtrTy}, {}) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll index 29c320da6c0a7..f8cd868a4c755 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll @@ -37,15 +37,6 @@ define i64 @strict_align_feature(ptr %p) #0 { attributes #0 = { "target-features"="+strict-align" } -; FALLBACK-WITH-REPORT-ERR: remark: :0:0: unable to translate instruction: call -; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for direct_mem -; FALLBACK-WITH-REPORT-OUT-LABEL: direct_mem -define void @direct_mem(i32 %x, i32 %y) { -entry: - tail call void asm sideeffect "", "imr,imr,~{memory}"(i32 %x, i32 %y) - ret void -} - ; FALLBACK-WITH-REPORT-ERR: remark: :0:0: unable to lower function{{.*}}scalable_arg ; FALLBACK-WITH-REPORT-OUT-LABEL: scalable_arg define @scalable_arg( %pred, ptr %addr) #1 { diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll index 42f6570047fc7..c5e0d1ac4b33a 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll @@ -258,3 +258,88 @@ define i64 @test_input_with_matching_constraint_to_physical_register() { %1 = tail call i64 asm "", "={x2},0"(i64 0) ret i64 %1 } + +define void @test_indirectify_i32_value(i32 %x, i32 %y) { + ; CHECK-LABEL: name: test_indirectify_i32_value + ; CHECK: bb.1.entry: + ; CHECK-NEXT: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1 + ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: G_STORE [[COPY]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %stack.0) + ; CHECK-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1 + ; CHECK-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX1]](p0) :: (store (s32) into %stack.1) + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[FRAME_INDEX]](p0), 262158 /* mem:m */, [[FRAME_INDEX1]](p0) + ; CHECK-NEXT: RET_ReallyLR +entry: + tail call void asm sideeffect "", "imr,imr,~{memory}"(i32 %x, i32 %y) + ret void +} + +define void @test_indirectify_i32_constant() { + ; CHECK-LABEL: name: test_indirectify_i32_constant + ; CHECK: bb.1.entry: + ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p0) = G_CONSTANT_POOL %const.0 + ; CHECK-NEXT: [[CONSTANT_POOL1:%[0-9]+]]:_(p0) = G_CONSTANT_POOL %const.1 + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[CONSTANT_POOL]](p0), 262158 /* mem:m */, [[CONSTANT_POOL1]](p0) + ; CHECK-NEXT: RET_ReallyLR +entry: + tail call void asm sideeffect "", "imr,imr,~{memory}"(i32 42, i32 0) + ret void +} + +define void @test_indirectify_i16_value(i16 %val) { + ; CHECK-LABEL: name: test_indirectify_i16_value + ; CHECK: bb.1.entry: + ; CHECK-NEXT: liveins: $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) + ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: G_STORE [[TRUNC]](s16), [[FRAME_INDEX]](p0) :: (store (s16) into %stack.0) + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[FRAME_INDEX]](p0) + ; CHECK-NEXT: RET_ReallyLR +entry: + tail call void asm sideeffect "", "imr,~{memory}"(i16 %val) + ret void +} + +define void @test_indirectify_i16_constant() { + ; CHECK-LABEL: name: test_indirectify_i16_constant + ; CHECK: bb.1.entry: + ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p0) = G_CONSTANT_POOL %const.0 + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[CONSTANT_POOL]](p0) + ; CHECK-NEXT: RET_ReallyLR +entry: + tail call void asm sideeffect "", "imr,~{memory}"(i16 42) + ret void +} + +define void @test_indirectify_i64_value(i64 %val) { + ; CHECK-LABEL: name: test_indirectify_i64_value + ; CHECK: bb.1.entry: + ; CHECK-NEXT: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: G_STORE [[COPY]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %stack.0) + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[FRAME_INDEX]](p0) + ; CHECK-NEXT: RET_ReallyLR +entry: + tail call void asm sideeffect "", "imr,~{memory}"(i64 %val) + ret void +} + +define void @test_indirectify_i64_constant() { + ; CHECK-LABEL: name: test_indirectify_i64_constant + ; CHECK: bb.1.entry: + ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p0) = G_CONSTANT_POOL %const.0 + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[CONSTANT_POOL]](p0) + ; CHECK-NEXT: RET_ReallyLR +entry: + tail call void asm sideeffect "", "imr,~{memory}"(i64 42) + ret void +} + +; TODO: add more types diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll index 2cde060529bec..80379ada0390b 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll @@ -331,6 +331,92 @@ define amdgpu_kernel void @asm_constraint_n_n() { ret void } +define void @test_indirectify_i32_value(i32 %x, i32 %y) { + ; CHECK-LABEL: name: test_indirectify_i32_value + ; CHECK: bb.1.entry: + ; CHECK-NEXT: liveins: $vgpr0, $vgpr1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: G_STORE [[COPY]](s32), [[FRAME_INDEX]](p5) :: (store (s32) into %stack.0, addrspace 5) + ; CHECK-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.1 + ; CHECK-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX1]](p5) :: (store (s32) into %stack.1, addrspace 5) + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[FRAME_INDEX]](p5), 262158 /* mem:m */, [[FRAME_INDEX1]](p5) + ; CHECK-NEXT: SI_RETURN +entry: + tail call void asm sideeffect "", "imr,imr,~{memory}"(i32 %x, i32 %y) + ret void +} + +define void @test_indirectify_i32_constant() { + ; CHECK-LABEL: name: test_indirectify_i32_constant + ; CHECK: bb.1.entry: + ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p1) = G_CONSTANT_POOL %const.0 + ; CHECK-NEXT: [[CONSTANT_POOL1:%[0-9]+]]:_(p1) = G_CONSTANT_POOL %const.1 + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[CONSTANT_POOL]](p1), 262158 /* mem:m */, [[CONSTANT_POOL1]](p1) + ; CHECK-NEXT: SI_RETURN +entry: + tail call void asm sideeffect "", "imr,imr,~{memory}"(i32 42, i32 0) + ret void +} + + +define void @test_indirectify_i16_value(i16 %val) { + ; CHECK-LABEL: name: test_indirectify_i16_value + ; CHECK: bb.1.entry: + ; CHECK-NEXT: liveins: $vgpr0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) + ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: G_STORE [[TRUNC]](s16), [[FRAME_INDEX]](p5) :: (store (s16) into %stack.0, addrspace 5) + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[FRAME_INDEX]](p5) + ; CHECK-NEXT: SI_RETURN +entry: + tail call void asm sideeffect "", "imr,~{memory}"(i16 %val) + ret void +} + +define void @test_indirectify_i16_constant() { + ; CHECK-LABEL: name: test_indirectify_i16_constant + ; CHECK: bb.1.entry: + ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p1) = G_CONSTANT_POOL %const.0 + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[CONSTANT_POOL]](p1) + ; CHECK-NEXT: SI_RETURN +entry: + tail call void asm sideeffect "", "imr,~{memory}"(i16 42) + ret void +} + +define void @test_indirectify_i64_value(i64 %val) { + ; CHECK-LABEL: name: test_indirectify_i64_value + ; CHECK: bb.1.entry: + ; CHECK-NEXT: liveins: $vgpr0, $vgpr1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32) + ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: G_STORE [[MV]](s64), [[FRAME_INDEX]](p5) :: (store (s64) into %stack.0, addrspace 5) + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[FRAME_INDEX]](p5) + ; CHECK-NEXT: SI_RETURN +entry: + tail call void asm sideeffect "", "imr,~{memory}"(i64 %val) + ret void +} + +define void @test_indirectify_i64_constant() { + ; CHECK-LABEL: name: test_indirectify_i64_constant + ; CHECK: bb.1.entry: + ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p1) = G_CONSTANT_POOL %const.0 + ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[CONSTANT_POOL]](p1) + ; CHECK-NEXT: SI_RETURN +entry: + tail call void asm sideeffect "", "imr,~{memory}"(i64 42) + ret void +} + !llvm.module.flags = !{!1} !0 = !{i32 70} !1 = !{i32 1, !"amdhsa_code_object_version", i32 500}