Skip to content

Commit

Permalink
ARMLoadStoreOptimizer: Fix errata 602117 handling and make testcase a…
Browse files Browse the repository at this point in the history 
…ctually test for it

This fixes PR23912

Differential Revision: http://reviews.llvm.org/D10620

llvm-svn: 240582
  • Loading branch information
@MatzeB
MatzeB committed Jun 24, 2015
1 parent a681ea2 commit ba3ecc3
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 122 deletions.
219 changes: 112 additions & 107 deletions llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
View file
Expand Up @@ -1464,119 +1464,124 @@ bool ARMLoadStoreOpt::FixInvalidRegPairOp(MachineBasicBlock &MBB,
MachineBasicBlock::iterator &MBBI) {
MachineInstr *MI = &*MBBI;
unsigned Opcode = MI->getOpcode();
if (Opcode == ARM::LDRD || Opcode == ARM::STRD) {
const MachineOperand &BaseOp = MI->getOperand(2);
unsigned BaseReg = BaseOp.getReg();
unsigned EvenReg = MI->getOperand(0).getReg();
unsigned OddReg = MI->getOperand(1).getReg();
unsigned EvenRegNum = TRI->getDwarfRegNum(EvenReg, false);
unsigned OddRegNum = TRI->getDwarfRegNum(OddReg, false);
// ARM errata 602117: LDRD with base in list may result in incorrect base
// register when interrupted or faulted.
bool Errata602117 = EvenReg == BaseReg && STI->isCortexM3();
if (!Errata602117 &&
((EvenRegNum & 1) == 0 && (EvenRegNum + 1) == OddRegNum))
return false;
if (Opcode != ARM::LDRD && Opcode != ARM::STRD && Opcode != ARM::t2LDRDi8)
return false;

MachineBasicBlock::iterator NewBBI = MBBI;
bool isT2 = Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8;
bool isLd = Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8;
bool EvenDeadKill = isLd ?
MI->getOperand(0).isDead() : MI->getOperand(0).isKill();
bool EvenUndef = MI->getOperand(0).isUndef();
bool OddDeadKill = isLd ?
MI->getOperand(1).isDead() : MI->getOperand(1).isKill();
bool OddUndef = MI->getOperand(1).isUndef();
bool BaseKill = BaseOp.isKill();
bool BaseUndef = BaseOp.isUndef();
bool OffKill = isT2 ? false : MI->getOperand(3).isKill();
bool OffUndef = isT2 ? false : MI->getOperand(3).isUndef();
int OffImm = getMemoryOpOffset(MI);
unsigned PredReg = 0;
ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);

if (OddRegNum > EvenRegNum && OffImm == 0) {
// Ascending register numbers and no offset. It's safe to change it to a
// ldm or stm.
unsigned NewOpc = (isLd)
? (isT2 ? ARM::t2LDMIA : ARM::LDMIA)
: (isT2 ? ARM::t2STMIA : ARM::STMIA);
if (isLd) {
BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
.addReg(BaseReg, getKillRegState(BaseKill))
.addImm(Pred).addReg(PredReg)
.addReg(EvenReg, getDefRegState(isLd) | getDeadRegState(EvenDeadKill))
.addReg(OddReg, getDefRegState(isLd) | getDeadRegState(OddDeadKill));
++NumLDRD2LDM;
} else {
BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
.addReg(BaseReg, getKillRegState(BaseKill))
.addImm(Pred).addReg(PredReg)
.addReg(EvenReg,
getKillRegState(EvenDeadKill) | getUndefRegState(EvenUndef))
.addReg(OddReg,
getKillRegState(OddDeadKill) | getUndefRegState(OddUndef));
++NumSTRD2STM;
}
const MachineOperand &BaseOp = MI->getOperand(2);
unsigned BaseReg = BaseOp.getReg();
unsigned EvenReg = MI->getOperand(0).getReg();
unsigned OddReg = MI->getOperand(1).getReg();
unsigned EvenRegNum = TRI->getDwarfRegNum(EvenReg, false);
unsigned OddRegNum = TRI->getDwarfRegNum(OddReg, false);

// ARM errata 602117: LDRD with base in list may result in incorrect base
// register when interrupted or faulted.
bool Errata602117 = EvenReg == BaseReg &&
(Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8) && STI->isCortexM3();
// ARM LDRD/STRD needs consecutive registers.
bool NonConsecutiveRegs = (Opcode == ARM::LDRD || Opcode == ARM::STRD) &&
(EvenRegNum % 2 != 0 || EvenRegNum + 1 != OddRegNum);

if (!Errata602117 && !NonConsecutiveRegs)
return false;

MachineBasicBlock::iterator NewBBI = MBBI;
bool isT2 = Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8;
bool isLd = Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8;
bool EvenDeadKill = isLd ?
MI->getOperand(0).isDead() : MI->getOperand(0).isKill();
bool EvenUndef = MI->getOperand(0).isUndef();
bool OddDeadKill = isLd ?
MI->getOperand(1).isDead() : MI->getOperand(1).isKill();
bool OddUndef = MI->getOperand(1).isUndef();
bool BaseKill = BaseOp.isKill();
bool BaseUndef = BaseOp.isUndef();
bool OffKill = isT2 ? false : MI->getOperand(3).isKill();
bool OffUndef = isT2 ? false : MI->getOperand(3).isUndef();
int OffImm = getMemoryOpOffset(MI);
unsigned PredReg = 0;
ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);

if (OddRegNum > EvenRegNum && OffImm == 0) {
// Ascending register numbers and no offset. It's safe to change it to a
// ldm or stm.
unsigned NewOpc = (isLd)
? (isT2 ? ARM::t2LDMIA : ARM::LDMIA)
: (isT2 ? ARM::t2STMIA : ARM::STMIA);
if (isLd) {
BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
.addReg(BaseReg, getKillRegState(BaseKill))
.addImm(Pred).addReg(PredReg)
.addReg(EvenReg, getDefRegState(isLd) | getDeadRegState(EvenDeadKill))
.addReg(OddReg, getDefRegState(isLd) | getDeadRegState(OddDeadKill));
++NumLDRD2LDM;
} else {
BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
.addReg(BaseReg, getKillRegState(BaseKill))
.addImm(Pred).addReg(PredReg)
.addReg(EvenReg,
getKillRegState(EvenDeadKill) | getUndefRegState(EvenUndef))
.addReg(OddReg,
getKillRegState(OddDeadKill) | getUndefRegState(OddUndef));
++NumSTRD2STM;
}
NewBBI = std::prev(MBBI);
} else {
// Split into two instructions.
unsigned NewOpc = (isLd)
? (isT2 ? (OffImm < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
: (isT2 ? (OffImm < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
// Be extra careful for thumb2. t2LDRi8 can't reference a zero offset,
// so adjust and use t2LDRi12 here for that.
unsigned NewOpc2 = (isLd)
? (isT2 ? (OffImm+4 < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
: (isT2 ? (OffImm+4 < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
DebugLoc dl = MBBI->getDebugLoc();
// If this is a load and base register is killed, it may have been
// re-defed by the load, make sure the first load does not clobber it.
if (isLd &&
(BaseKill || OffKill) &&
(TRI->regsOverlap(EvenReg, BaseReg))) {
assert(!TRI->regsOverlap(OddReg, BaseReg));
InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2,
OddReg, OddDeadKill, false,
BaseReg, false, BaseUndef, false, OffUndef,
Pred, PredReg, TII, isT2);
NewBBI = std::prev(MBBI);
InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc,
EvenReg, EvenDeadKill, false,
BaseReg, BaseKill, BaseUndef, OffKill, OffUndef,
Pred, PredReg, TII, isT2);
} else {
// Split into two instructions.
unsigned NewOpc = (isLd)
? (isT2 ? (OffImm < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
: (isT2 ? (OffImm < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
// Be extra careful for thumb2. t2LDRi8 can't reference a zero offset,
// so adjust and use t2LDRi12 here for that.
unsigned NewOpc2 = (isLd)
? (isT2 ? (OffImm+4 < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
: (isT2 ? (OffImm+4 < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
DebugLoc dl = MBBI->getDebugLoc();
// If this is a load and base register is killed, it may have been
// re-defed by the load, make sure the first load does not clobber it.
if (isLd &&
(BaseKill || OffKill) &&
(TRI->regsOverlap(EvenReg, BaseReg))) {
assert(!TRI->regsOverlap(OddReg, BaseReg));
InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2,
OddReg, OddDeadKill, false,
BaseReg, false, BaseUndef, false, OffUndef,
Pred, PredReg, TII, isT2);
NewBBI = std::prev(MBBI);
InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc,
EvenReg, EvenDeadKill, false,
BaseReg, BaseKill, BaseUndef, OffKill, OffUndef,
Pred, PredReg, TII, isT2);
} else {
if (OddReg == EvenReg && EvenDeadKill) {
// If the two source operands are the same, the kill marker is
// probably on the first one. e.g.
// t2STRDi8 %R5<kill>, %R5, %R9<kill>, 0, 14, %reg0
EvenDeadKill = false;
OddDeadKill = true;
}
// Never kill the base register in the first instruction.
if (EvenReg == BaseReg)
EvenDeadKill = false;
InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc,
EvenReg, EvenDeadKill, EvenUndef,
BaseReg, false, BaseUndef, false, OffUndef,
Pred, PredReg, TII, isT2);
NewBBI = std::prev(MBBI);
InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2,
OddReg, OddDeadKill, OddUndef,
BaseReg, BaseKill, BaseUndef, OffKill, OffUndef,
Pred, PredReg, TII, isT2);
if (OddReg == EvenReg && EvenDeadKill) {
// If the two source operands are the same, the kill marker is
// probably on the first one. e.g.
// t2STRDi8 %R5<kill>, %R5, %R9<kill>, 0, 14, %reg0
EvenDeadKill = false;
OddDeadKill = true;
}
if (isLd)
++NumLDRD2LDR;
else
++NumSTRD2STR;
// Never kill the base register in the first instruction.
if (EvenReg == BaseReg)
EvenDeadKill = false;
InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc,
EvenReg, EvenDeadKill, EvenUndef,
BaseReg, false, BaseUndef, false, OffUndef,
Pred, PredReg, TII, isT2);
NewBBI = std::prev(MBBI);
InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2,
OddReg, OddDeadKill, OddUndef,
BaseReg, BaseKill, BaseUndef, OffKill, OffUndef,
Pred, PredReg, TII, isT2);
}

MBB.erase(MI);
MBBI = NewBBI;
return true;
if (isLd)
++NumLDRD2LDR;
else
++NumSTRD2STR;
}
return false;

MBB.erase(MI);
MBBI = NewBBI;
return true;
}

/// An optimization pass to turn multiple LDR / STR ops of the same base and
Expand Down
29 changes: 15 additions & 14 deletions llvm/test/CodeGen/ARM/ldrd.ll
View file
Expand Up @@ -6,23 +6,24 @@

; Magic ARM pair hints works best with linearscan / fast.

; Cortex-M3 errata 602117: LDRD with base in list may result in incorrect base
; register when interrupted or faulted.

@b = external global i64*

define i64 @t(i64 %a) nounwind readonly {
entry:
; A8-LABEL: t:
; A8: ldrd r2, r3, [r2]

; M3-LABEL: t:
; M3-NOT: ldrd
; We use the following two to force values into specific registers.
declare i64* @get_ptr()
declare void @use_i64(i64 %v)

%0 = load i64*, i64** @b, align 4
%1 = load i64, i64* %0, align 4
%2 = mul i64 %1, %a
ret i64 %2
define void @test_ldrd(i64 %a) nounwind readonly {
; CHECK-LABEL: test_ldrd:
; CHECK: bl{{x?}} _get_ptr
; A8: ldrd r0, r1, [r0]
; Cortex-M3 errata 602117: LDRD with base in list may result in incorrect base
; register when interrupted or faulted.
; M3-NOT: ldrd r[[REGNUM:[0-9]+]], {{r[0-9]+}}, [r[[REGNUM]]]
; CHECK: bl{{x?}} _use_i64
%ptr = call i64* @get_ptr()
%v = load i64, i64* %ptr, align 8
call void @use_i64(i64 %v)
ret void
}

; rdar://10435045 mixed LDRi8/LDRi12
Expand Down
2 changes: 1 addition & 1 deletion llvm/test/CodeGen/Thumb2/float-ops.ll
View file
Expand Up @@ -109,7 +109,7 @@ entry:
define double @load_d(double* %a) {
entry:
; CHECK-LABEL: load_d:
; NONE: ldrd r0, r1, [r0]
; NONE: ldm r0, {r0, r1}
; HARD: vldr d0, [r0]
%0 = load double, double* %a, align 8
ret double %0
Expand Down

0 comments on commit ba3ecc3

Please sign in to comment.