diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 45fc6b0238877..2089bfba5ff37 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -230,6 +230,7 @@ class AArch64InstructionSelector : public InstructionSelector { bool selectMOPS(MachineInstr &I, MachineRegisterInfo &MRI); bool selectUSMovFromExtend(MachineInstr &I, MachineRegisterInfo &MRI); + bool selectIndexedExtLoad(MachineInstr &I, MachineRegisterInfo &MRI); bool selectIndexedLoad(MachineInstr &I, MachineRegisterInfo &MRI); bool selectIndexedStore(GIndexedStore &I, MachineRegisterInfo &MRI); @@ -3047,6 +3048,9 @@ bool AArch64InstructionSelector::select(MachineInstr &I) { return constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI); } + case TargetOpcode::G_INDEXED_ZEXTLOAD: + case TargetOpcode::G_INDEXED_SEXTLOAD: + return selectIndexedExtLoad(I, MRI); case TargetOpcode::G_INDEXED_LOAD: return selectIndexedLoad(I, MRI); case TargetOpcode::G_INDEXED_STORE: @@ -5648,6 +5652,93 @@ MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImmFP( return &*Mov; } +bool AArch64InstructionSelector::selectIndexedExtLoad( + MachineInstr &MI, MachineRegisterInfo &MRI) { + auto &ExtLd = cast(MI); + Register Dst = ExtLd.getDstReg(); + Register WriteBack = ExtLd.getWritebackReg(); + Register Base = ExtLd.getBaseReg(); + Register Offset = ExtLd.getOffsetReg(); + LLT Ty = MRI.getType(Dst); + assert(Ty.getSizeInBits() <= 64); // Only for scalar GPRs. + unsigned MemSizeBits = ExtLd.getMMO().getMemoryType().getSizeInBits(); + bool IsPre = ExtLd.isPre(); + bool IsSExt = isa(ExtLd); + bool InsertIntoXReg = false; + bool IsDst64 = Ty.getSizeInBits() == 64; + + unsigned Opc = 0; + LLT NewLdDstTy; + LLT s32 = LLT::scalar(32); + LLT s64 = LLT::scalar(64); + + if (MemSizeBits == 8) { + if (IsSExt) { + if (IsDst64) + Opc = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost; + else + Opc = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost; + NewLdDstTy = IsDst64 ? s64 : s32; + } else { + Opc = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost; + InsertIntoXReg = IsDst64; + NewLdDstTy = s32; + } + } else if (MemSizeBits == 16) { + if (IsSExt) { + if (IsDst64) + Opc = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost; + else + Opc = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost; + NewLdDstTy = IsDst64 ? s64 : s32; + } else { + Opc = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost; + InsertIntoXReg = IsDst64; + NewLdDstTy = s32; + } + } else if (MemSizeBits == 32) { + if (IsSExt) { + Opc = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost; + NewLdDstTy = s64; + } else { + Opc = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost; + InsertIntoXReg = IsDst64; + NewLdDstTy = s32; + } + } else { + llvm_unreachable("Unexpected size for indexed load"); + } + + if (RBI.getRegBank(Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID) + return false; // We should be on gpr. + + auto Cst = getIConstantVRegVal(Offset, MRI); + if (!Cst) + return false; // Shouldn't happen, but just in case. + + auto LdMI = MIB.buildInstr(Opc, {WriteBack, NewLdDstTy}, {Base}) + .addImm(Cst->getSExtValue()); + LdMI.cloneMemRefs(ExtLd); + constrainSelectedInstRegOperands(*LdMI, TII, TRI, RBI); + // Make sure to select the load with the MemTy as the dest type, and then + // insert into X reg if needed. + if (InsertIntoXReg) { + // Generate a SUBREG_TO_REG. + auto SubToReg = MIB.buildInstr(TargetOpcode::SUBREG_TO_REG, {Dst}, {}) + .addImm(0) + .addUse(LdMI.getReg(1)) + .addImm(AArch64::sub_32); + RBI.constrainGenericRegister(SubToReg.getReg(0), AArch64::GPR64RegClass, + MRI); + } else { + auto Copy = MIB.buildCopy(Dst, LdMI.getReg(1)); + selectCopy(*Copy, TII, MRI, TRI, RBI); + } + MI.eraseFromParent(); + + return true; +} + bool AArch64InstructionSelector::selectIndexedLoad(MachineInstr &MI, MachineRegisterInfo &MRI) { // TODO: extending loads. diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 9fadc008de206..226cdee44ece2 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -459,7 +459,24 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) .legalIf(IndexedLoadBasicPred) .unsupported(); getActionDefinitionsBuilder({G_INDEXED_SEXTLOAD, G_INDEXED_ZEXTLOAD}) - .unsupported(); // TODO: implement + .unsupportedIf( + atomicOrderingAtLeastOrStrongerThan(0, AtomicOrdering::Unordered)) + .legalIf(all(typeInSet(0, {s16, s32, s64}), + LegalityPredicate([=](const LegalityQuery &Q) { + LLT LdTy = Q.Types[0]; + LLT PtrTy = Q.Types[1]; + LLT MemTy = Q.MMODescrs[0].MemoryTy; + if (PtrTy != p0) + return false; + if (LdTy == s16) + return MemTy == s8; + if (LdTy == s32) + return MemTy == s8 || MemTy == s16; + if (LdTy == s64) + return MemTy == s8 || MemTy == s16 || MemTy == s32; + return false; + }))) + .unsupported(); // Constants getActionDefinitionsBuilder(G_CONSTANT) diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp index b74c4021d3e4e..b956cd4782b79 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp @@ -887,9 +887,12 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { break; } break; - case TargetOpcode::G_INDEXED_LOAD: case TargetOpcode::G_INDEXED_SEXTLOAD: - case TargetOpcode::G_INDEXED_ZEXTLOAD: { + case TargetOpcode::G_INDEXED_ZEXTLOAD: + // These should always be GPR. + OpRegBankIdx[0] = PMI_FirstGPR; + break; + case TargetOpcode::G_INDEXED_LOAD: { if (isLoadFromFPType(MI)) OpRegBankIdx[0] = PMI_FirstFPR; break; diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-indexed-load-stores.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-indexed-load-stores.mir index bd0317ec6a136..e207d3a94dc5a 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-indexed-load-stores.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-indexed-load-stores.mir @@ -109,3 +109,83 @@ body: | $x0 = COPY %writeback RET_ReallyLR implicit $x0 ... +--- +name: post_zextload_s8_to_s64 +body: | + bb.0: + liveins: $x0 + + ; CHECK-LABEL: name: post_zextload_s8_to_s64 + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %ptr:_(p0) = COPY $x0 + ; CHECK-NEXT: %offset:_(s64) = G_CONSTANT i64 8 + ; CHECK-NEXT: %dst:_(s64), %writeback:_(p0) = G_INDEXED_ZEXTLOAD %ptr, %offset(s64), 0 :: (load (s8), align 8) + ; CHECK-NEXT: $x0 = COPY %writeback(p0) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %ptr:_(p0) = COPY $x0 + %offset:_(s64) = G_CONSTANT i64 8 + %dst:_(s64), %writeback:_(p0) = G_INDEXED_ZEXTLOAD %ptr, %offset, 0 :: (load (s8), align 8) + $x0 = COPY %writeback + RET_ReallyLR implicit $x0 +... +--- +name: post_sextload_s8_to_s64 +body: | + bb.0: + liveins: $x0 + + ; CHECK-LABEL: name: post_sextload_s8_to_s64 + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %ptr:_(p0) = COPY $x0 + ; CHECK-NEXT: %offset:_(s64) = G_CONSTANT i64 8 + ; CHECK-NEXT: %dst:_(s64), %writeback:_(p0) = G_INDEXED_SEXTLOAD %ptr, %offset(s64), 0 :: (load (s8), align 8) + ; CHECK-NEXT: $x0 = COPY %writeback(p0) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %ptr:_(p0) = COPY $x0 + %offset:_(s64) = G_CONSTANT i64 8 + %dst:_(s64), %writeback:_(p0) = G_INDEXED_SEXTLOAD %ptr, %offset, 0 :: (load (s8), align 8) + $x0 = COPY %writeback + RET_ReallyLR implicit $x0 +... +--- +name: post_sextload_s32_to_s64 +body: | + bb.0: + liveins: $x0 + + ; CHECK-LABEL: name: post_sextload_s32_to_s64 + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %ptr:_(p0) = COPY $x0 + ; CHECK-NEXT: %offset:_(s64) = G_CONSTANT i64 8 + ; CHECK-NEXT: %dst:_(s64), %writeback:_(p0) = G_INDEXED_SEXTLOAD %ptr, %offset(s64), 0 :: (load (s32), align 8) + ; CHECK-NEXT: $x0 = COPY %writeback(p0) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %ptr:_(p0) = COPY $x0 + %offset:_(s64) = G_CONSTANT i64 8 + %dst:_(s64), %writeback:_(p0) = G_INDEXED_SEXTLOAD %ptr, %offset, 0 :: (load (s32), align 8) + $x0 = COPY %writeback + RET_ReallyLR implicit $x0 +... +--- +name: post_zextload_s32_to_s64 +body: | + bb.0: + liveins: $x0 + + ; CHECK-LABEL: name: post_zextload_s32_to_s64 + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %ptr:_(p0) = COPY $x0 + ; CHECK-NEXT: %offset:_(s64) = G_CONSTANT i64 8 + ; CHECK-NEXT: %dst:_(s64), %writeback:_(p0) = G_INDEXED_ZEXTLOAD %ptr, %offset(s64), 0 :: (load (s32), align 8) + ; CHECK-NEXT: $x0 = COPY %writeback(p0) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %ptr:_(p0) = COPY $x0 + %offset:_(s64) = G_CONSTANT i64 8 + %dst:_(s64), %writeback:_(p0) = G_INDEXED_ZEXTLOAD %ptr, %offset, 0 :: (load (s32), align 8) + $x0 = COPY %writeback + RET_ReallyLR implicit $x0 +... diff --git a/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll b/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll index dc8cbd1e43b90..d949f95209577 100644 --- a/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll +++ b/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll @@ -503,24 +503,11 @@ define ptr @preidx8zext64(ptr %src, ptr %out) { } define ptr @preidx32sext64(ptr %src, ptr %out) { -; CHECK64-LABEL: preidx32sext64: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsw x8, [x0, #4]! -; CHECK64-NEXT: str x8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: preidx32sext64: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsw x8, [x0, #4] -; GISEL-NEXT: add x0, x0, #4 -; GISEL-NEXT: str x8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: preidx32sext64: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsw x8, [x0, #4]! -; CHECK32-NEXT: str x8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: preidx32sext64: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsw x8, [x0, #4]! +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret %ptr = getelementptr inbounds i32, ptr %src, i64 1 %tmp = load i32, ptr %ptr, align 4 %ext = sext i32 %tmp to i64 @@ -529,24 +516,11 @@ define ptr @preidx32sext64(ptr %src, ptr %out) { } define ptr @preidx16sext32(ptr %src, ptr %out) { -; CHECK64-LABEL: preidx16sext32: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsh w8, [x0, #2]! -; CHECK64-NEXT: str w8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: preidx16sext32: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsh w8, [x0, #2] -; GISEL-NEXT: add x0, x0, #2 -; GISEL-NEXT: str w8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: preidx16sext32: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsh w8, [x0, #2]! -; CHECK32-NEXT: str w8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: preidx16sext32: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsh w8, [x0, #2]! +; CHECK-NEXT: str w8, [x1] +; CHECK-NEXT: ret %ptr = getelementptr inbounds i16, ptr %src, i64 1 %tmp = load i16, ptr %ptr, align 4 %ext = sext i16 %tmp to i32 @@ -555,24 +529,11 @@ define ptr @preidx16sext32(ptr %src, ptr %out) { } define ptr @preidx16sext64(ptr %src, ptr %out) { -; CHECK64-LABEL: preidx16sext64: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsh x8, [x0, #2]! -; CHECK64-NEXT: str x8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: preidx16sext64: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsh x8, [x0, #2] -; GISEL-NEXT: add x0, x0, #2 -; GISEL-NEXT: str x8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: preidx16sext64: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsh x8, [x0, #2]! -; CHECK32-NEXT: str x8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: preidx16sext64: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsh x8, [x0, #2]! +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret %ptr = getelementptr inbounds i16, ptr %src, i64 1 %tmp = load i16, ptr %ptr, align 4 %ext = sext i16 %tmp to i64 @@ -581,24 +542,11 @@ define ptr @preidx16sext64(ptr %src, ptr %out) { } define ptr @preidx8sext32(ptr %src, ptr %out) { -; CHECK64-LABEL: preidx8sext32: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsb w8, [x0, #1]! -; CHECK64-NEXT: str w8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: preidx8sext32: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsb w8, [x0, #1] -; GISEL-NEXT: add x0, x0, #1 -; GISEL-NEXT: str w8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: preidx8sext32: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsb w8, [x0, #1]! -; CHECK32-NEXT: str w8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: preidx8sext32: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsb w8, [x0, #1]! +; CHECK-NEXT: str w8, [x1] +; CHECK-NEXT: ret %ptr = getelementptr inbounds i8, ptr %src, i64 1 %tmp = load i8, ptr %ptr, align 4 %ext = sext i8 %tmp to i32 @@ -607,24 +555,11 @@ define ptr @preidx8sext32(ptr %src, ptr %out) { } define ptr @preidx8sext64(ptr %src, ptr %out) { -; CHECK64-LABEL: preidx8sext64: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsb x8, [x0, #1]! -; CHECK64-NEXT: str x8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: preidx8sext64: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsb x8, [x0, #1] -; GISEL-NEXT: add x0, x0, #1 -; GISEL-NEXT: str x8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: preidx8sext64: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsb x8, [x0, #1]! -; CHECK32-NEXT: str x8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: preidx8sext64: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsb x8, [x0, #1]! +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret %ptr = getelementptr inbounds i8, ptr %src, i64 1 %tmp = load i8, ptr %ptr, align 4 %ext = sext i8 %tmp to i64 @@ -662,24 +597,11 @@ define ptr @postidx_clobber(ptr %addr) nounwind noinline ssp { } define ptr @preidx32_sb(ptr %src, ptr %out) { -; CHECK64-LABEL: preidx32_sb: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsb w8, [x0, #1]! -; CHECK64-NEXT: str w8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: preidx32_sb: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsb w8, [x0, #1] -; GISEL-NEXT: add x0, x0, #1 -; GISEL-NEXT: str w8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: preidx32_sb: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsb w8, [x0, #1]! -; CHECK32-NEXT: str w8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: preidx32_sb: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsb w8, [x0, #1]! +; CHECK-NEXT: str w8, [x1] +; CHECK-NEXT: ret %ptr = getelementptr inbounds i8, ptr %src, i64 1 %tmp = load i8, ptr %ptr, align 1 %sext = sext i8 %tmp to i32 @@ -688,24 +610,11 @@ define ptr @preidx32_sb(ptr %src, ptr %out) { } define ptr @preidx32_sh(ptr %src, ptr %out) { -; CHECK64-LABEL: preidx32_sh: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsh w8, [x0, #2]! -; CHECK64-NEXT: str w8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: preidx32_sh: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsh w8, [x0, #2] -; GISEL-NEXT: add x0, x0, #2 -; GISEL-NEXT: str w8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: preidx32_sh: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsh w8, [x0, #2]! -; CHECK32-NEXT: str w8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: preidx32_sh: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsh w8, [x0, #2]! +; CHECK-NEXT: str w8, [x1] +; CHECK-NEXT: ret %ptr = getelementptr inbounds i16, ptr %src, i64 1 %tmp = load i16, ptr %ptr, align 2 %sext = sext i16 %tmp to i32 @@ -714,24 +623,11 @@ define ptr @preidx32_sh(ptr %src, ptr %out) { } define ptr @preidx64_sb(ptr %src, ptr %out) { -; CHECK64-LABEL: preidx64_sb: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsb x8, [x0, #1]! -; CHECK64-NEXT: str x8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: preidx64_sb: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsb x8, [x0, #1] -; GISEL-NEXT: add x0, x0, #1 -; GISEL-NEXT: str x8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: preidx64_sb: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsb x8, [x0, #1]! -; CHECK32-NEXT: str x8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: preidx64_sb: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsb x8, [x0, #1]! +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret %ptr = getelementptr inbounds i8, ptr %src, i64 1 %tmp = load i8, ptr %ptr, align 1 %sext = sext i8 %tmp to i64 @@ -740,24 +636,11 @@ define ptr @preidx64_sb(ptr %src, ptr %out) { } define ptr @preidx64_sh(ptr %src, ptr %out) { -; CHECK64-LABEL: preidx64_sh: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsh x8, [x0, #2]! -; CHECK64-NEXT: str x8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: preidx64_sh: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsh x8, [x0, #2] -; GISEL-NEXT: add x0, x0, #2 -; GISEL-NEXT: str x8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: preidx64_sh: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsh x8, [x0, #2]! -; CHECK32-NEXT: str x8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: preidx64_sh: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsh x8, [x0, #2]! +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret %ptr = getelementptr inbounds i16, ptr %src, i64 1 %tmp = load i16, ptr %ptr, align 2 %sext = sext i16 %tmp to i64 @@ -766,24 +649,11 @@ define ptr @preidx64_sh(ptr %src, ptr %out) { } define ptr @preidx64_sw(ptr %src, ptr %out) { -; CHECK64-LABEL: preidx64_sw: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsw x8, [x0, #4]! -; CHECK64-NEXT: str x8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: preidx64_sw: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsw x8, [x0, #4] -; GISEL-NEXT: add x0, x0, #4 -; GISEL-NEXT: str x8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: preidx64_sw: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsw x8, [x0, #4]! -; CHECK32-NEXT: str x8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: preidx64_sw: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsw x8, [x0, #4]! +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret %ptr = getelementptr inbounds i32, ptr %src, i64 1 %tmp = load i32, ptr %ptr, align 2 %sext = sext i32 %tmp to i64 @@ -792,24 +662,11 @@ define ptr @preidx64_sw(ptr %src, ptr %out) { } define ptr @postidx32_sb(ptr %src, ptr %out) { -; CHECK64-LABEL: postidx32_sb: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsb w8, [x0], #1 -; CHECK64-NEXT: str w8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: postidx32_sb: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsb w8, [x0] -; GISEL-NEXT: add x0, x0, #1 -; GISEL-NEXT: str w8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: postidx32_sb: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsb w8, [x0], #1 -; CHECK32-NEXT: str w8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: postidx32_sb: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsb w8, [x0], #1 +; CHECK-NEXT: str w8, [x1] +; CHECK-NEXT: ret %tmp = load i8, ptr %src, align 1 %ptr = getelementptr inbounds i8, ptr %src, i64 1 %sext = sext i8 %tmp to i32 @@ -818,24 +675,11 @@ define ptr @postidx32_sb(ptr %src, ptr %out) { } define ptr @postidx32_sh(ptr %src, ptr %out) { -; CHECK64-LABEL: postidx32_sh: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsh w8, [x0], #2 -; CHECK64-NEXT: str w8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: postidx32_sh: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsh w8, [x0] -; GISEL-NEXT: add x0, x0, #2 -; GISEL-NEXT: str w8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: postidx32_sh: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsh w8, [x0], #2 -; CHECK32-NEXT: str w8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: postidx32_sh: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsh w8, [x0], #2 +; CHECK-NEXT: str w8, [x1] +; CHECK-NEXT: ret %tmp = load i16, ptr %src, align 2 %ptr = getelementptr inbounds i16, ptr %src, i64 1 %sext = sext i16 %tmp to i32 @@ -844,24 +688,11 @@ define ptr @postidx32_sh(ptr %src, ptr %out) { } define ptr @postidx64_sb(ptr %src, ptr %out) { -; CHECK64-LABEL: postidx64_sb: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsb x8, [x0], #1 -; CHECK64-NEXT: str x8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: postidx64_sb: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsb x8, [x0] -; GISEL-NEXT: add x0, x0, #1 -; GISEL-NEXT: str x8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: postidx64_sb: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsb x8, [x0], #1 -; CHECK32-NEXT: str x8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: postidx64_sb: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsb x8, [x0], #1 +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret %tmp = load i8, ptr %src, align 1 %ptr = getelementptr inbounds i8, ptr %src, i64 1 %sext = sext i8 %tmp to i64 @@ -870,24 +701,11 @@ define ptr @postidx64_sb(ptr %src, ptr %out) { } define ptr @postidx64_sh(ptr %src, ptr %out) { -; CHECK64-LABEL: postidx64_sh: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsh x8, [x0], #2 -; CHECK64-NEXT: str x8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: postidx64_sh: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsh x8, [x0] -; GISEL-NEXT: add x0, x0, #2 -; GISEL-NEXT: str x8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: postidx64_sh: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsh x8, [x0], #2 -; CHECK32-NEXT: str x8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: postidx64_sh: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsh x8, [x0], #2 +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret %tmp = load i16, ptr %src, align 2 %ptr = getelementptr inbounds i16, ptr %src, i64 1 %sext = sext i16 %tmp to i64 @@ -896,24 +714,11 @@ define ptr @postidx64_sh(ptr %src, ptr %out) { } define ptr @postidx64_sw(ptr %src, ptr %out) { -; CHECK64-LABEL: postidx64_sw: -; CHECK64: ; %bb.0: -; CHECK64-NEXT: ldrsw x8, [x0], #4 -; CHECK64-NEXT: str x8, [x1] -; CHECK64-NEXT: ret -; -; GISEL-LABEL: postidx64_sw: -; GISEL: ; %bb.0: -; GISEL-NEXT: ldrsw x8, [x0] -; GISEL-NEXT: add x0, x0, #4 -; GISEL-NEXT: str x8, [x1] -; GISEL-NEXT: ret -; -; CHECK32-LABEL: postidx64_sw: -; CHECK32: ; %bb.0: -; CHECK32-NEXT: ldrsw x8, [x0], #4 -; CHECK32-NEXT: str x8, [x1] -; CHECK32-NEXT: ret +; CHECK-LABEL: postidx64_sw: +; CHECK: ; %bb.0: +; CHECK-NEXT: ldrsw x8, [x0], #4 +; CHECK-NEXT: str x8, [x1] +; CHECK-NEXT: ret %tmp = load i32, ptr %src, align 4 %ptr = getelementptr inbounds i32, ptr %src, i64 1 %sext = sext i32 %tmp to i64