-
Notifications
You must be signed in to change notification settings - Fork 4.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Implement ARM32 atomic intrinsics #97792
base: main
Are you sure you want to change the base?
Changes from all commits
71b7344
0ee75a8
2638041
2e5cdd3
1d24a02
62311c4
9910a94
a6985d1
62f6a2c
745f9f7
81d3174
178471a
1b03761
ba65197
d78f840
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -61,6 +61,11 @@ bool CodeGen::genInstrWithConstant( | |
{ | ||
case INS_add: | ||
case INS_sub: | ||
if (imm < 0) | ||
{ | ||
imm = -imm; | ||
ins = (ins == INS_add) ? INS_sub : INS_add; | ||
} | ||
immFitsInIns = validImmForInstr(ins, (target_ssize_t)imm, flags); | ||
break; | ||
|
||
|
@@ -653,6 +658,254 @@ void CodeGen::genJumpTable(GenTree* treeNode) | |
genProduceReg(treeNode); | ||
} | ||
|
||
//------------------------------------------------------------------------ | ||
// genLockedInstructions: Generate code for a GT_XADD or GT_XCHG node. | ||
// | ||
// Arguments: | ||
// treeNode - the GT_XADD/XCHG node | ||
// | ||
void CodeGen::genLockedInstructions(GenTreeOp* treeNode) | ||
{ | ||
GenTree* data = treeNode->AsOp()->gtOp2; | ||
GenTree* addr = treeNode->AsOp()->gtOp1; | ||
regNumber targetReg = treeNode->GetRegNum(); | ||
regNumber dataReg = data->GetRegNum(); | ||
regNumber addrReg = addr->GetRegNum(); | ||
|
||
genConsumeAddress(addr); | ||
genConsumeRegs(data); | ||
|
||
assert(!treeNode->OperIs(GT_XORR, GT_XAND)); | ||
assert(treeNode->OperIs(GT_XCHG) || !varTypeIsSmall(treeNode->TypeGet())); | ||
|
||
emitAttr dataSize = emitActualTypeSize(data); | ||
|
||
regNumber tempReg = treeNode->ExtractTempReg(RBM_ALLINT); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We now use new data structure to extract the temp register in #101647. Please update it accordingly. |
||
regNumber storeReg = (treeNode->OperGet() == GT_XCHG) ? dataReg : treeNode->ExtractTempReg(RBM_ALLINT); | ||
regNumber loadReg = (targetReg != REG_NA) ? targetReg : storeReg; | ||
|
||
// Check allocator assumptions | ||
// | ||
// The register allocator should have extended the lifetimes of all input and internal registers so that | ||
// none interfere with the target. | ||
noway_assert(addrReg != targetReg); | ||
|
||
noway_assert(addrReg != loadReg); | ||
noway_assert(dataReg != loadReg); | ||
|
||
noway_assert((treeNode->OperGet() == GT_XCHG) || (addrReg != dataReg)); | ||
|
||
assert(addr->isUsedFromReg()); | ||
noway_assert(tempReg != REG_NA); | ||
noway_assert(tempReg != targetReg); | ||
noway_assert((targetReg != REG_NA) || (treeNode->OperGet() != GT_XCHG)); | ||
|
||
// Store exclusive unpredictable cases must be avoided | ||
noway_assert(tempReg != addrReg); | ||
|
||
// NOTE: `genConsumeAddress` marks the consumed register as not a GC pointer, as it assumes that the input | ||
// registers | ||
// die at the first instruction generated by the node. This is not the case for these atomics as the input | ||
// registers are multiply-used. As such, we need to mark the addr register as containing a GC pointer until | ||
// we are finished generating the code for this node. | ||
|
||
gcInfo.gcMarkRegPtrVal(addrReg, addr->TypeGet()); | ||
|
||
// Emit code like this: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I know this is done for other platforms as well, but not sure why this cannot be done in lower. The codegen should just emit the code instead of inserting the loop like code here. |
||
// retry: | ||
// ldrex loadReg, [addrReg] | ||
// add storeReg, loadReg, dataReg # Only for GT_XADD | ||
// # GT_XCHG storeReg === dataReg | ||
// strex tempReg, storeReg, [addrReg] | ||
// cmp tempReg, 0 | ||
// bne retry | ||
// dmb ish | ||
|
||
instruction insLd = INS_ldrex; | ||
instruction insSt = INS_strex; | ||
if (varTypeIsByte(treeNode->TypeGet())) | ||
{ | ||
insLd = INS_ldrexb; | ||
insSt = INS_strexb; | ||
} | ||
else if (varTypeIsShort(treeNode->TypeGet())) | ||
{ | ||
insLd = INS_ldrexh; | ||
insSt = INS_strexh; | ||
} | ||
|
||
instGen_MemoryBarrier(); | ||
|
||
BasicBlock* labelRetry = genCreateTempLabel(); | ||
genDefineTempLabel(labelRetry); | ||
|
||
// The following instruction includes a acquire half barrier | ||
GetEmitter()->emitIns_R_R(insLd, dataSize, loadReg, addrReg); | ||
|
||
if (treeNode->OperGet() == GT_XADD) | ||
{ | ||
if (data->isContainedIntOrIImmed()) | ||
{ | ||
genInstrWithConstant(INS_add, dataSize, storeReg, loadReg, data->AsIntConCommon()->IconValue(), | ||
INS_FLAGS_DONT_CARE, tempReg); | ||
} | ||
else | ||
{ | ||
GetEmitter()->emitIns_R_R_R(INS_add, dataSize, storeReg, loadReg, dataReg); | ||
} | ||
} | ||
|
||
// The following instruction includes a release half barrier | ||
GetEmitter()->emitIns_R_R_R(insSt, dataSize, tempReg, storeReg, addrReg); | ||
|
||
GetEmitter()->emitIns_R_I(INS_cmp, EA_4BYTE, tempReg, 0); | ||
GetEmitter()->emitIns_J(INS_bne, labelRetry); | ||
|
||
instGen_MemoryBarrier(); | ||
|
||
gcInfo.gcMarkRegSetNpt(addr->gtGetRegMask()); | ||
|
||
if (targetReg != REG_NA) | ||
{ | ||
if (varTypeIsSmall(treeNode->TypeGet()) && varTypeIsSigned(treeNode->TypeGet())) | ||
{ | ||
instruction mov = varTypeIsShort(treeNode->TypeGet()) ? INS_sxth : INS_sxtb; | ||
GetEmitter()->emitIns_Mov(mov, EA_4BYTE, targetReg, targetReg, /* canSkip */ false); | ||
} | ||
|
||
genProduceReg(treeNode); | ||
} | ||
} | ||
|
||
//------------------------------------------------------------------------ | ||
// genCodeForCmpXchg: Produce code for a GT_CMPXCHG node. | ||
// | ||
// Arguments: | ||
// tree - the GT_CMPXCHG node | ||
// | ||
void CodeGen::genCodeForCmpXchg(GenTreeCmpXchg* treeNode) | ||
{ | ||
assert(treeNode->OperIs(GT_CMPXCHG)); | ||
|
||
GenTree* addr = treeNode->Addr(); // arg1 | ||
GenTree* data = treeNode->Data(); // arg2 | ||
GenTree* comparand = treeNode->Comparand(); // arg3 | ||
|
||
regNumber targetReg = treeNode->GetRegNum(); | ||
regNumber dataReg = data->GetRegNum(); | ||
regNumber addrReg = addr->GetRegNum(); | ||
regNumber comparandReg = comparand->GetRegNum(); | ||
|
||
genConsumeAddress(addr); | ||
genConsumeRegs(data); | ||
genConsumeRegs(comparand); | ||
|
||
emitAttr dataSize = emitActualTypeSize(data); | ||
|
||
regNumber exResultReg = treeNode->ExtractTempReg(RBM_ALLINT); | ||
|
||
// Check allocator assumptions | ||
// | ||
// The register allocator should have extended the lifetimes of all input and internal registers so that | ||
// none interfere with the target. | ||
noway_assert(addrReg != targetReg); | ||
noway_assert(dataReg != targetReg); | ||
noway_assert(comparandReg != targetReg); | ||
noway_assert(addrReg != dataReg); | ||
noway_assert(targetReg != REG_NA); | ||
noway_assert(exResultReg != REG_NA); | ||
noway_assert(exResultReg != targetReg); | ||
|
||
assert(addr->isUsedFromReg()); | ||
assert(data->isUsedFromReg()); | ||
assert(!comparand->isUsedFromMemory()); | ||
|
||
// Store exclusive unpredictable cases must be avoided | ||
noway_assert(exResultReg != dataReg); | ||
noway_assert(exResultReg != addrReg); | ||
|
||
// NOTE: `genConsumeAddress` marks the consumed register as not a GC pointer, as it assumes that the input | ||
// registers | ||
// die at the first instruction generated by the node. This is not the case for these atomics as the input | ||
// registers are multiply-used. As such, we need to mark the addr register as containing a GC pointer until | ||
// we are finished generating the code for this node. | ||
|
||
gcInfo.gcMarkRegPtrVal(addrReg, addr->TypeGet()); | ||
|
||
// Emit code like this: | ||
// retry: | ||
// ldrex targetReg, [addrReg] | ||
// cmp targetReg, comparandReg | ||
// bne compareFail | ||
// strex exResult, dataReg, [addrReg] | ||
// cmp exResult, 0 | ||
// bne retry | ||
// compareFail: | ||
// dmb ish | ||
|
||
instruction insLd = INS_ldrex; | ||
instruction insSt = INS_strex; | ||
if (varTypeIsByte(treeNode->TypeGet())) | ||
{ | ||
insLd = INS_ldrexb; | ||
insSt = INS_strexb; | ||
} | ||
else if (varTypeIsShort(treeNode->TypeGet())) | ||
{ | ||
insLd = INS_ldrexh; | ||
insSt = INS_strexh; | ||
} | ||
|
||
instGen_MemoryBarrier(); | ||
|
||
BasicBlock* labelRetry = genCreateTempLabel(); | ||
BasicBlock* labelCompareFail = genCreateTempLabel(); | ||
genDefineTempLabel(labelRetry); | ||
|
||
// The following instruction includes a acquire half barrier | ||
GetEmitter()->emitIns_R_R(insLd, dataSize, targetReg, addrReg); | ||
|
||
if (comparand->isContainedIntOrIImmed()) | ||
{ | ||
if (comparand->IsIntegralConst(0) && emitter::isLowRegister(targetReg)) | ||
{ | ||
GetEmitter()->emitIns_J_R(INS_cbnz, EA_4BYTE, labelCompareFail, targetReg); | ||
} | ||
else | ||
{ | ||
assert(comparand->AsIntConCommon()->IconValue() <= INT32_MAX); | ||
GetEmitter()->emitIns_R_I(INS_cmp, EA_4BYTE, targetReg, | ||
(target_ssize_t)comparand->AsIntConCommon()->IconValue()); | ||
GetEmitter()->emitIns_J(INS_bne, labelCompareFail); | ||
} | ||
} | ||
else | ||
{ | ||
GetEmitter()->emitIns_R_R(INS_cmp, EA_4BYTE, targetReg, comparandReg); | ||
GetEmitter()->emitIns_J(INS_bne, labelCompareFail); | ||
} | ||
|
||
// The following instruction includes a release half barrier | ||
GetEmitter()->emitIns_R_R_R(insSt, dataSize, exResultReg, dataReg, addrReg); | ||
|
||
GetEmitter()->emitIns_R_I(INS_cmp, EA_4BYTE, exResultReg, 0); | ||
GetEmitter()->emitIns_J(INS_bne, labelRetry); | ||
|
||
genDefineTempLabel(labelCompareFail); | ||
|
||
instGen_MemoryBarrier(); | ||
|
||
gcInfo.gcMarkRegSetNpt(addr->gtGetRegMask()); | ||
|
||
if (varTypeIsSmall(treeNode->TypeGet()) && varTypeIsSigned(treeNode->TypeGet())) | ||
{ | ||
instruction mov = varTypeIsShort(treeNode->TypeGet()) ? INS_sxth : INS_sxtb; | ||
GetEmitter()->emitIns_Mov(mov, EA_4BYTE, targetReg, targetReg, /* canSkip */ false); | ||
} | ||
|
||
genProduceReg(treeNode); | ||
} | ||
|
||
//------------------------------------------------------------------------ | ||
// genGetInsForOper: Return instruction encoding of the operation tree. | ||
// | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
genLocked*
methods added here looks similar to the one in arm64. can the logic be shared and have just a single method incodegenarmarch.cpp
?