From 941d4cd14ce5845e47b467708d25631abef3c785 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 29 Apr 2024 20:22:21 +0200 Subject: [PATCH] JIT: Move internal reserved registers to a side table (#101647) This gets rid of `GenTree::gtRsvdRegs` by moving internal registers to a side table. We generally use internal registers very rarely, so making the lookup more costly seems worth the trade off (especially to make it easier to expand `regMaskTP` to 16 bytes). There was one exception where we used internal registers a lot, which was `GT_CALL` for R2R codegen on arm64/arm32. For those nodes we always allocate an internal register to load the target into (the target is obtained by loading the R2R indirection cell that is passed in an argument register). For arm64 it was simple to avoid this internal register: we can simply use LR always, since that register is going to be overwritten by the call anyway. This results in -2% TP for crossgen2 arm64 just from avoiding building this extra interval. This is also the cause of the asm diffs. For arm32 the same strategy doesn't work as well because loading into LR is a 4 byte instruction while loading into other registers is a 2 byte instruction. So for arm32 we still use an internal register and take the small throughput hit. This change reduces JIT memory usage by ~1.5%. The throughput cost (when discounting some spurious inlining decision changes) seems to be around 0.1%. --- src/coreclr/jit/codegenarm.cpp | 24 ++--- src/coreclr/jit/codegenarm64.cpp | 45 ++++---- src/coreclr/jit/codegenarmarch.cpp | 82 ++++++++------- src/coreclr/jit/codegencommon.cpp | 107 ++++++++++++++++++++ src/coreclr/jit/codegeninterface.h | 22 +++- src/coreclr/jit/codegenlinear.cpp | 2 +- src/coreclr/jit/codegenloongarch64.cpp | 38 +++---- src/coreclr/jit/codegenriscv64.cpp | 76 +++++++------- src/coreclr/jit/codegenxarch.cpp | 66 ++++++------ src/coreclr/jit/emitarm.cpp | 18 ++-- src/coreclr/jit/emitarm64.cpp | 8 +- src/coreclr/jit/emitloongarch64.cpp | 6 +- src/coreclr/jit/emitriscv64.cpp | 14 +-- src/coreclr/jit/emitxarch.cpp | 2 +- src/coreclr/jit/gentree.cpp | 76 ++------------ src/coreclr/jit/gentree.h | 6 -- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 6 +- src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 44 ++++---- src/coreclr/jit/jithashtable.h | 50 +++++++-- src/coreclr/jit/lsra.cpp | 4 +- src/coreclr/jit/lsraarmarch.cpp | 20 ++-- src/coreclr/jit/lsrabuild.cpp | 4 - src/coreclr/jit/simdcodegenxarch.cpp | 6 +- 23 files changed, 416 insertions(+), 310 deletions(-) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 2c010f116a265..dea8b19fbee94 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -280,7 +280,7 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types targetType, GenTre if (targetType == TYP_FLOAT) { // Get a temp integer register - regNumber tmpReg = tree->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(tree); float f = forceCastToFloat(constValue); instGen_Set_Reg_To_Imm(EA_4BYTE, tmpReg, *((int*)(&f))); @@ -293,8 +293,8 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types targetType, GenTre unsigned* cv = (unsigned*)&constValue; // Get two temp integer registers - regNumber tmpReg1 = tree->ExtractTempReg(); - regNumber tmpReg2 = tree->GetSingleTempReg(); + regNumber tmpReg1 = internalRegisters.Extract(tree); + regNumber tmpReg2 = internalRegisters.GetSingle(tree); instGen_Set_Reg_To_Imm(EA_4BYTE, tmpReg1, cv[0]); instGen_Set_Reg_To_Imm(EA_4BYTE, tmpReg2, cv[1]); @@ -431,9 +431,9 @@ void CodeGen::genLclHeap(GenTree* tree) } // Setup the regTmp, if there is one. - if (tree->AvailableTempRegCount() > 0) + if (internalRegisters.Count(tree) > 0) { - regTmp = tree->ExtractTempReg(); + regTmp = internalRegisters.Extract(tree); } // If we have an outgoing arg area then we must adjust the SP by popping off the @@ -833,7 +833,7 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) gcInfo.gcMarkRegPtrVal(REG_WRITE_BARRIER_DST_BYREF, dstAddr->TypeGet()); // Temp register used to perform the sequence of loads and stores. - regNumber tmpReg = cpObjNode->ExtractTempReg(); + regNumber tmpReg = internalRegisters.Extract(cpObjNode); assert(genIsValidIntReg(tmpReg)); if (cpObjNode->IsVolatile()) @@ -1026,18 +1026,18 @@ void CodeGen::genCodeForStoreLclFld(GenTreeLclFld* tree) { // Arm supports unaligned access only for integer types, // convert the storing floating data into 1 or 2 integer registers and write them as int. - regNumber addr = tree->ExtractTempReg(); + regNumber addr = internalRegisters.Extract(tree); emit->emitIns_R_S(INS_lea, EA_PTRSIZE, addr, varNum, offset); if (targetType == TYP_FLOAT) { - regNumber floatAsInt = tree->GetSingleTempReg(); + regNumber floatAsInt = internalRegisters.GetSingle(tree); emit->emitIns_Mov(INS_vmov_f2i, EA_4BYTE, floatAsInt, dataReg, /* canSkip */ false); emit->emitIns_R_R(INS_str, EA_4BYTE, floatAsInt, addr); } else { - regNumber halfdoubleAsInt1 = tree->ExtractTempReg(); - regNumber halfdoubleAsInt2 = tree->GetSingleTempReg(); + regNumber halfdoubleAsInt1 = internalRegisters.Extract(tree); + regNumber halfdoubleAsInt2 = internalRegisters.GetSingle(tree); emit->emitIns_R_R_R(INS_vmov_d2i, EA_8BYTE, halfdoubleAsInt1, halfdoubleAsInt2, dataReg); emit->emitIns_R_R_I(INS_str, EA_4BYTE, halfdoubleAsInt1, addr, 0); emit->emitIns_R_R_I(INS_str, EA_4BYTE, halfdoubleAsInt1, addr, 4); @@ -1209,7 +1209,7 @@ void CodeGen::genCkfinite(GenTree* treeNode) emitter* emit = GetEmitter(); var_types targetType = treeNode->TypeGet(); - regNumber intReg = treeNode->GetSingleTempReg(); + regNumber intReg = internalRegisters.GetSingle(treeNode); regNumber fpReg = genConsumeReg(treeNode->AsOp()->gtOp1); regNumber targetReg = treeNode->GetRegNum(); @@ -1592,7 +1592,7 @@ void CodeGen::genFloatToIntCast(GenTree* treeNode) genConsumeOperands(treeNode->AsOp()); - regNumber tmpReg = treeNode->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(treeNode); assert(insVcvt != INS_invalid); GetEmitter()->emitIns_R_R(insVcvt, dstSize, tmpReg, op1->GetRegNum()); diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 8695545cc934a..dc79220dcd0b8 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2372,7 +2372,7 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types targetType, GenTre else { // Get a temp integer register to compute long address. - regNumber addrReg = tree->GetSingleTempReg(); + regNumber addrReg = internalRegisters.GetSingle(tree); // We must load the FP constant from the constant pool // Emit a data section constant for the float or double constant. @@ -2407,7 +2407,7 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types targetType, GenTre else { // Get a temp integer register to compute long address. - regNumber addrReg = tree->GetSingleTempReg(); + regNumber addrReg = internalRegisters.GetSingle(tree); simd8_t constValue; memcpy(&constValue, &vecCon->gtSimdVal, sizeof(simd8_t)); @@ -2431,7 +2431,7 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types targetType, GenTre else { // Get a temp integer register to compute long address. - regNumber addrReg = tree->GetSingleTempReg(); + regNumber addrReg = internalRegisters.GetSingle(tree); simd16_t constValue = {}; memcpy(&constValue, &vecCon->gtSimdVal, sizeof(simd12_t)); @@ -2455,7 +2455,7 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types targetType, GenTre else { // Get a temp integer register to compute long address. - regNumber addrReg = tree->GetSingleTempReg(); + regNumber addrReg = internalRegisters.GetSingle(tree); simd16_t constValue; memcpy(&constValue, &vecCon->gtSimdVal, sizeof(simd16_t)); @@ -3132,12 +3132,12 @@ void CodeGen::genLclHeap(GenTree* tree) // since we don't need any internal registers. if (compiler->info.compInitMem) { - assert(tree->AvailableTempRegCount() == 0); + assert(internalRegisters.Count(tree) == 0); regCnt = targetReg; } else { - regCnt = tree->ExtractTempReg(); + regCnt = internalRegisters.Extract(tree); inst_Mov(size->TypeGet(), regCnt, targetReg, /* canSkip */ true); } @@ -3254,12 +3254,12 @@ void CodeGen::genLclHeap(GenTree* tree) assert(regCnt == REG_NA); if (compiler->info.compInitMem) { - assert(tree->AvailableTempRegCount() == 0); + assert(internalRegisters.Count(tree) == 0); regCnt = targetReg; } else { - regCnt = tree->ExtractTempReg(); + regCnt = internalRegisters.Extract(tree); } instGen_Set_Reg_To_Imm(((unsigned int)amount == amount) ? EA_4BYTE : EA_8BYTE, regCnt, amount); } @@ -3323,7 +3323,7 @@ void CodeGen::genLclHeap(GenTree* tree) // // Setup the regTmp - regNumber regTmp = tree->GetSingleTempReg(); + regNumber regTmp = internalRegisters.GetSingle(tree); BasicBlock* loop = genCreateTempLabel(); BasicBlock* done = genCreateTempLabel(); @@ -3668,7 +3668,7 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) unsigned slots = layout->GetSlotCount(); // Temp register(s) used to perform the sequence of loads and stores. - regNumber tmpReg = cpObjNode->ExtractTempReg(RBM_ALLINT); + regNumber tmpReg = internalRegisters.Extract(cpObjNode, RBM_ALLINT); regNumber tmpReg2 = REG_NA; assert(genIsValidIntReg(tmpReg)); @@ -3677,7 +3677,7 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) if (slots > 1) { - tmpReg2 = cpObjNode->ExtractTempReg(RBM_ALLINT); + tmpReg2 = internalRegisters.Extract(cpObjNode, RBM_ALLINT); assert(tmpReg2 != tmpReg); assert(genIsValidIntReg(tmpReg2)); assert(tmpReg2 != REG_WRITE_BARRIER_DST_BYREF); @@ -3730,8 +3730,8 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) regNumber tmpSimdReg2 = REG_NA; if ((slots >= 4) && compiler->IsBaselineSimdIsaSupported()) { - tmpSimdReg1 = cpObjNode->ExtractTempReg(RBM_ALLFLOAT); - tmpSimdReg2 = cpObjNode->ExtractTempReg(RBM_ALLFLOAT); + tmpSimdReg1 = internalRegisters.Extract(cpObjNode, RBM_ALLFLOAT); + tmpSimdReg2 = internalRegisters.Extract(cpObjNode, RBM_ALLFLOAT); } unsigned i = 0; @@ -3810,7 +3810,7 @@ void CodeGen::genTableBasedSwitch(GenTree* treeNode) regNumber idxReg = treeNode->AsOp()->gtOp1->GetRegNum(); regNumber baseReg = treeNode->AsOp()->gtOp2->GetRegNum(); - regNumber tmpReg = treeNode->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(treeNode); // load the ip-relative offset (which is relative to start of fgFirstBB) GetEmitter()->emitIns_R_R_R(INS_ldr, EA_4BYTE, baseReg, baseReg, idxReg, INS_OPTS_LSL); @@ -3869,7 +3869,7 @@ void CodeGen::genLockedInstructions(GenTreeOp* treeNode) case GT_XAND: { // Grab a temp reg to perform `MVN` for dataReg first. - regNumber tempReg = treeNode->GetSingleTempReg(); + regNumber tempReg = internalRegisters.GetSingle(treeNode); GetEmitter()->emitIns_R_R(INS_mvn, dataSize, tempReg, dataReg); GetEmitter()->emitIns_R_R_R(INS_ldclral, dataSize, tempReg, (targetReg == REG_NA) ? REG_ZR : targetReg, addrReg); @@ -3902,9 +3902,10 @@ void CodeGen::genLockedInstructions(GenTreeOp* treeNode) // These are imported normally if Atomics aren't supported. assert(!treeNode->OperIs(GT_XORR, GT_XAND)); - regNumber exResultReg = treeNode->ExtractTempReg(RBM_ALLINT); - regNumber storeDataReg = (treeNode->OperGet() == GT_XCHG) ? dataReg : treeNode->ExtractTempReg(RBM_ALLINT); - regNumber loadReg = (targetReg != REG_NA) ? targetReg : storeDataReg; + regNumber exResultReg = internalRegisters.Extract(treeNode, RBM_ALLINT); + regNumber storeDataReg = + (treeNode->OperGet() == GT_XCHG) ? dataReg : internalRegisters.Extract(treeNode, RBM_ALLINT); + regNumber loadReg = (targetReg != REG_NA) ? targetReg : storeDataReg; // Check allocator assumptions // @@ -4055,7 +4056,7 @@ void CodeGen::genCodeForCmpXchg(GenTreeCmpXchg* treeNode) } else { - regNumber exResultReg = treeNode->ExtractTempReg(RBM_ALLINT); + regNumber exResultReg = internalRegisters.Extract(treeNode, RBM_ALLINT); // Check allocator assumptions // @@ -4600,7 +4601,7 @@ void CodeGen::genCkfinite(GenTree* treeNode) emitter* emit = GetEmitter(); // Extract exponent into a register. - regNumber intReg = treeNode->GetSingleTempReg(); + regNumber intReg = internalRegisters.GetSingle(treeNode); regNumber fpReg = genConsumeReg(op1); inst_Mov(targetType, intReg, fpReg, /* canSkip */ false, emitActualTypeSize(treeNode)); @@ -5351,7 +5352,7 @@ void CodeGen::genStoreIndTypeSimd12(GenTreeStoreInd* treeNode) regNumber dataReg = genConsumeReg(data); // Need an additional integer register to extract upper 4 bytes from data. - regNumber tmpReg = treeNode->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(treeNode); // 8-byte write GetEmitter()->emitIns_R_R(INS_str, EA_8BYTE, dataReg, addrReg); @@ -5386,7 +5387,7 @@ void CodeGen::genLoadIndTypeSimd12(GenTreeIndir* treeNode) regNumber addrReg = genConsumeReg(addr); // Need an additional int register to read upper 4 bytes, which is different from targetReg - regNumber tmpReg = treeNode->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(treeNode); // 8-byte read GetEmitter()->emitIns_R_R(INS_ldr, EA_8BYTE, tgtReg, addrReg); diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index d015332a76d8b..90447292dbe83 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -910,9 +910,9 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) // in ARM64/ARM // Setup loReg (and hiReg) from the internal registers that we reserved in lower. // - regNumber loReg = treeNode->ExtractTempReg(); + regNumber loReg = internalRegisters.Extract(treeNode); #ifdef TARGET_ARM64 - regNumber hiReg = treeNode->GetSingleTempReg(); + regNumber hiReg = internalRegisters.GetSingle(treeNode); #endif // TARGET_ARM64 GenTreeLclVarCommon* srcLclNode = nullptr; @@ -1268,7 +1268,7 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode) regNumber allocatedValueReg = REG_NA; if (treeNode->gtNumRegs == 1) { - allocatedValueReg = treeNode->ExtractTempReg(); + allocatedValueReg = internalRegisters.Extract(treeNode); } // Pick a register to store intermediate values in for the to-stack @@ -1640,19 +1640,19 @@ void CodeGen::genCodeForLclFld(GenTreeLclFld* tree) { // Arm supports unaligned access only for integer types, // load the floating data as 1 or 2 integer registers and convert them to float. - regNumber addr = tree->ExtractTempReg(); + regNumber addr = internalRegisters.Extract(tree); emit->emitIns_R_S(INS_lea, EA_PTRSIZE, addr, varNum, offs); if (targetType == TYP_FLOAT) { - regNumber floatAsInt = tree->GetSingleTempReg(); + regNumber floatAsInt = internalRegisters.GetSingle(tree); emit->emitIns_R_R(INS_ldr, EA_4BYTE, floatAsInt, addr); emit->emitIns_Mov(INS_vmov_i2f, EA_4BYTE, targetReg, floatAsInt, /* canSkip */ false); } else { - regNumber halfdoubleAsInt1 = tree->ExtractTempReg(); - regNumber halfdoubleAsInt2 = tree->GetSingleTempReg(); + regNumber halfdoubleAsInt1 = internalRegisters.Extract(tree); + regNumber halfdoubleAsInt2 = internalRegisters.GetSingle(tree); emit->emitIns_R_R_I(INS_ldr, EA_4BYTE, halfdoubleAsInt1, addr, 0); emit->emitIns_R_R_I(INS_ldr, EA_4BYTE, halfdoubleAsInt2, addr, 4); emit->emitIns_R_R_R(INS_vmov_i2d, EA_8BYTE, targetReg, halfdoubleAsInt1, halfdoubleAsInt2); @@ -1694,7 +1694,7 @@ void CodeGen::genCodeForIndexAddr(GenTreeIndexAddr* node) // The index is never contained, even if it is a constant. assert(index->isUsedFromReg()); - const regNumber tmpReg = node->ExtractTempReg(); + const regNumber tmpReg = internalRegisters.Extract(node); regNumber indexReg = index->GetRegNum(); @@ -1742,7 +1742,7 @@ void CodeGen::genCodeForIndexAddr(GenTreeIndexAddr* node) #ifdef TARGET_ARM64 if (!index->TypeIs(TYP_I_IMPL)) { - const regNumber tmpReg2 = node->ExtractTempReg(); + const regNumber tmpReg2 = internalRegisters.Extract(node); GetEmitter()->emitIns_Mov(INS_mov, EA_4BYTE, tmpReg2, indexReg, /* canSkip */ false); indexReg = tmpReg2; } @@ -2662,7 +2662,7 @@ void CodeGen::genCodeForInitBlkUnroll(GenTreeBlk* node) const int dstOffsetAdjustment = helper.GetDstOffset() - dstRegAddrAlignment; dstRegAddrAlignment = 0; - const regNumber tempReg = node->ExtractTempReg(RBM_ALLINT); + const regNumber tempReg = internalRegisters.Extract(node, RBM_ALLINT); genInstrWithConstant(INS_add, EA_PTRSIZE, tempReg, dstReg, dstOffsetAdjustment, tempReg); dstReg = tempReg; @@ -2684,7 +2684,7 @@ void CodeGen::genCodeForInitBlkUnroll(GenTreeBlk* node) if (shouldUse16ByteWideInstrs) { - const regNumber simdReg = node->GetSingleTempReg(RBM_ALLFLOAT); + const regNumber simdReg = internalRegisters.GetSingle(node, RBM_ALLFLOAT); const int initValue = (src->AsIntCon()->IconValue() & 0xFF); emit->emitIns_R_I(INS_movi, EA_16BYTE, simdReg, initValue, INS_OPTS_16B); @@ -2967,23 +2967,23 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) if ((srcOffsetAdjustment != 0) && (dstOffsetAdjustment != 0)) { - const regNumber tempReg1 = node->ExtractTempReg(RBM_ALLINT); + const regNumber tempReg1 = internalRegisters.Extract(node, RBM_ALLINT); genInstrWithConstant(INS_add, EA_PTRSIZE, tempReg1, srcReg, srcOffsetAdjustment, tempReg1); srcReg = tempReg1; - const regNumber tempReg2 = node->ExtractTempReg(RBM_ALLINT); + const regNumber tempReg2 = internalRegisters.Extract(node, RBM_ALLINT); genInstrWithConstant(INS_add, EA_PTRSIZE, tempReg2, dstReg, dstOffsetAdjustment, tempReg2); dstReg = tempReg2; } else if (srcOffsetAdjustment != 0) { - const regNumber tempReg = node->ExtractTempReg(RBM_ALLINT); + const regNumber tempReg = internalRegisters.Extract(node, RBM_ALLINT); genInstrWithConstant(INS_add, EA_PTRSIZE, tempReg, srcReg, srcOffsetAdjustment, tempReg); srcReg = tempReg; } else if (dstOffsetAdjustment != 0) { - const regNumber tempReg = node->ExtractTempReg(RBM_ALLINT); + const regNumber tempReg = internalRegisters.Extract(node, RBM_ALLINT); genInstrWithConstant(INS_add, EA_PTRSIZE, tempReg, dstReg, dstOffsetAdjustment, tempReg); dstReg = tempReg; } @@ -2991,16 +2991,16 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) regNumber intReg1 = REG_NA; regNumber intReg2 = REG_NA; - const unsigned intRegCount = node->AvailableTempRegCount(RBM_ALLINT); + const unsigned intRegCount = internalRegisters.Count(node, RBM_ALLINT); if (intRegCount >= 2) { - intReg1 = node->ExtractTempReg(RBM_ALLINT); - intReg2 = node->ExtractTempReg(RBM_ALLINT); + intReg1 = internalRegisters.Extract(node, RBM_ALLINT); + intReg2 = internalRegisters.Extract(node, RBM_ALLINT); } else if (intRegCount == 1) { - intReg1 = node->GetSingleTempReg(RBM_ALLINT); + intReg1 = internalRegisters.GetSingle(node, RBM_ALLINT); intReg2 = rsGetRsvdReg(); } else @@ -3010,8 +3010,8 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) if (shouldUse16ByteWideInstrs) { - const regNumber simdReg1 = node->ExtractTempReg(RBM_ALLFLOAT); - const regNumber simdReg2 = node->GetSingleTempReg(RBM_ALLFLOAT); + const regNumber simdReg1 = internalRegisters.Extract(node, RBM_ALLFLOAT); + const regNumber simdReg2 = internalRegisters.GetSingle(node, RBM_ALLFLOAT); helper.Unroll(FP_REGSIZE_BYTES, intReg1, simdReg1, simdReg2, srcReg, dstReg, GetEmitter()); } @@ -3022,7 +3022,7 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) #endif // TARGET_ARM64 #ifdef TARGET_ARM - const regNumber tempReg = node->ExtractTempReg(RBM_ALLINT); + const regNumber tempReg = internalRegisters.Extract(node, RBM_ALLINT); for (unsigned regSize = REGSIZE_BYTES; size > 0; size -= regSize, srcOffset += regSize, dstOffset += regSize) { @@ -3147,13 +3147,13 @@ void CodeGen::genCodeForMemmove(GenTreeBlk* tree) if (size >= simdSize) { // Number of SIMD regs needed to save the whole src to regs. - const unsigned numberOfSimdRegs = tree->AvailableTempRegCount(RBM_ALLFLOAT); + const unsigned numberOfSimdRegs = internalRegisters.Count(tree, RBM_ALLFLOAT); // Pop all temp regs to a local array, currently, this impl is limited with LSRA's MaxInternalCount regNumber tempRegs[LinearScan::MaxInternalCount] = {}; for (unsigned i = 0; i < numberOfSimdRegs; i++) { - tempRegs[i] = tree->ExtractTempReg(RBM_ALLFLOAT); + tempRegs[i] = internalRegisters.Extract(tree, RBM_ALLFLOAT); } auto emitSimdLoadStore = [&](bool load) { @@ -3190,15 +3190,15 @@ void CodeGen::genCodeForMemmove(GenTreeBlk* tree) const unsigned loadStoreSize = 1 << BitOperations::Log2(size); if (loadStoreSize == size) { - const regNumber tmpReg = tree->GetSingleTempReg(RBM_ALLINT); + const regNumber tmpReg = internalRegisters.GetSingle(tree, RBM_ALLINT); emitLoadStore(/* load */ true, loadStoreSize, tmpReg, 0); emitLoadStore(/* load */ false, loadStoreSize, tmpReg, 0); } else { - assert(tree->AvailableTempRegCount() == 2); - const regNumber tmpReg1 = tree->ExtractTempReg(RBM_ALLINT); - const regNumber tmpReg2 = tree->ExtractTempReg(RBM_ALLINT); + assert(internalRegisters.Count(tree) == 2); + const regNumber tmpReg1 = internalRegisters.Extract(tree, RBM_ALLINT); + const regNumber tmpReg2 = internalRegisters.Extract(tree, RBM_ALLINT); emitLoadStore(/* load */ true, loadStoreSize, tmpReg1, 0); emitLoadStore(/* load */ true, loadStoreSize, tmpReg2, size - loadStoreSize); emitLoadStore(/* load */ false, loadStoreSize, tmpReg1, 0); @@ -3258,7 +3258,7 @@ void CodeGen::genCodeForInitBlkLoop(GenTreeBlk* initBlkNode) // Extend liveness of dstReg in case if it gets killed by the store. gcInfo.gcMarkRegPtrVal(dstReg, dstNode->TypeGet()); - const regNumber offsetReg = initBlkNode->GetSingleTempReg(); + const regNumber offsetReg = internalRegisters.GetSingle(initBlkNode); instGen_Set_Reg_To_Imm(EA_PTRSIZE, offsetReg, size - TARGET_POINTER_SIZE); BasicBlock* loop = genCreateTempLabel(); @@ -3342,7 +3342,7 @@ void CodeGen::genCall(GenTreeCall* call) const regNumber regThis = genGetThisArgReg(call); #if defined(TARGET_ARM) - const regNumber tmpReg = call->ExtractTempReg(); + const regNumber tmpReg = internalRegisters.Extract(call); GetEmitter()->emitIns_R_R_I(INS_ldr, EA_4BYTE, tmpReg, regThis, 0); #elif defined(TARGET_ARM64) GetEmitter()->emitIns_R_R_I(INS_ldr, EA_4BYTE, REG_ZR, regThis, 0); @@ -3368,7 +3368,7 @@ void CodeGen::genCall(GenTreeCall* call) (call->IsVirtualStubRelativeIndir() && (call->gtEntryPoint.accessType == IAT_VALUE))); assert(call->gtControlExpr == nullptr); - regNumber tmpReg = call->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(call); // Register where we save call address in should not be overridden by epilog. assert((genRegMask(tmpReg) & (RBM_INT_CALLEE_TRASH & ~RBM_LR)) == genRegMask(tmpReg)); @@ -3376,7 +3376,7 @@ void CodeGen::genCall(GenTreeCall* call) call->IsVirtualStubRelativeIndir() ? compiler->virtualStubParamInfo->GetReg() : REG_R2R_INDIRECT_PARAM; GetEmitter()->emitIns_R_R(ins_Load(TYP_I_IMPL), emitActualTypeSize(TYP_I_IMPL), tmpReg, callAddrReg); // We will use this again when emitting the jump in genCallInstruction in the epilog - call->gtRsvdRegs |= genRegMask(tmpReg); + internalRegisters.Add(call, genRegMask(tmpReg)); } #endif @@ -3666,15 +3666,25 @@ void CodeGen::genCallInstruction(GenTreeCall* call) if (callThroughIndirReg != REG_NA) { assert(call->IsR2ROrVirtualStubRelativeIndir()); - regNumber targetAddrReg = call->GetSingleTempReg(); + regNumber targetAddrReg; // For fast tailcalls we have already loaded the call target when processing the call node. if (!call->IsFastTailCall()) { +#ifdef TARGET_ARM + // For arm32 we've allocated an internal register to load the target into. + // Loading into lr takes 4 bytes (instead of potentially 2 with another register). + targetAddrReg = internalRegisters.GetSingle(call); +#else + // For arm64 we just use lr and skip the internal register. + targetAddrReg = REG_LR; +#endif + GetEmitter()->emitIns_R_R(ins_Load(TYP_I_IMPL), emitActualTypeSize(TYP_I_IMPL), targetAddrReg, callThroughIndirReg); } else { + targetAddrReg = internalRegisters.GetSingle(call); // Register where we save call address in should not be overridden by epilog. assert((genRegMask(targetAddrReg) & (RBM_INT_CALLEE_TRASH & ~RBM_LR)) == genRegMask(targetAddrReg)); } @@ -3731,7 +3741,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) #ifdef TARGET_ARM if (!validImmForBL((ssize_t)addr)) { - regNumber tmpReg = call->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(call); instGen_Set_Reg_To_Imm(EA_HANDLE_CNS_RELOC, tmpReg, (ssize_t)addr); // clang-format off genEmitCall(emitter::EC_INDIR_R, @@ -4721,7 +4731,7 @@ void CodeGen::genLeaInstruction(GenTreeAddrMode* lea) if (offset != 0) { - regNumber tmpReg = lea->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(lea); // When generating fully interruptible code we have to use the "large offset" sequence // when calculating a EA_BYREF as we can't report a byref that points outside of the object @@ -4803,7 +4813,7 @@ void CodeGen::genLeaInstruction(GenTreeAddrMode* lea) else { // We require a tmpReg to hold the offset - regNumber tmpReg = lea->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(lea); // First load tmpReg with the large offset constant instGen_Set_Reg_To_Imm(EA_PTRSIZE, tmpReg, offset); diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 301b10b1de071..6ec296a1391bb 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -62,10 +62,117 @@ CodeGenInterface* getCodeGenerator(Compiler* comp) return new (comp, CMK_Codegen) CodeGen(comp); } +NodeInternalRegisters::NodeInternalRegisters(Compiler* comp) + : m_table(comp->getAllocator(CMK_LSRA)) +{ +} + +//------------------------------------------------------------------------ +// Add: Add internal allocated registers for the specified node. +// +// Parameters: +// tree - IR node to add internal allocated registers to +// regs - Registers to add +// +void NodeInternalRegisters::Add(GenTree* tree, regMaskTP regs) +{ + assert(regs != RBM_NONE); + + regMaskTP* result = m_table.LookupPointerOrAdd(tree, RBM_NONE); + *result |= regs; +} + +//------------------------------------------------------------------------ +// Extract: Find the lowest number temporary register from the gtRsvdRegs set +// that is also in the optional given mask (typically, RBM_ALLINT or +// RBM_ALLFLOAT), and return it. Remove this register from the temporary +// register set, so it won't be returned again. +// +// Parameters: +// tree - IR node whose internal registers to extract +// mask - Mask of allowed registers that can be returned +// +// Returns: +// Register number. +// +regNumber NodeInternalRegisters::Extract(GenTree* tree, regMaskTP mask) +{ + regMaskTP* regs = m_table.LookupPointer(tree); + assert(regs != nullptr); + + regMaskTP availableSet = *regs & mask; + assert(availableSet != RBM_NONE); + + regNumber result = genFirstRegNumFromMask(availableSet); + *regs ^= genRegMask(result); + + return result; +} + +//------------------------------------------------------------------------ +// GetSingleTempReg: There is expected to be exactly one available temporary register +// in the given mask in the internal register set. Get that register. No future calls to get +// a temporary register are expected. Removes the register from the set, but only in +// DEBUG to avoid doing unnecessary work in non-DEBUG builds. +// +// Parameters: +// tree - IR node whose internal registers to extract +// mask - Mask of allowed registers that can be returned +// +// Returns: +// Register number. +// +regNumber NodeInternalRegisters::GetSingle(GenTree* tree, regMaskTP mask) +{ + regMaskTP* regs = m_table.LookupPointer(tree); + assert(regs != nullptr); + + regMaskTP availableSet = *regs & mask; + assert(genExactlyOneBit(availableSet)); + + regNumber result = genFirstRegNumFromMask(availableSet); + INDEBUG(*regs &= ~genRegMask(result)); + + return result; +} + +//------------------------------------------------------------------------ +// GetAll: Get all internal registers for the specified IR node. +// +// Parameters: +// tree - IR node whose internal registers to query +// +// Returns: +// Mask of registers. +// +regMaskTP NodeInternalRegisters::GetAll(GenTree* tree) +{ + regMaskTP regs; + return m_table.Lookup(tree, ®s) ? regs : RBM_NONE; +} + +//------------------------------------------------------------------------ +// Count: return the number of available temporary registers in the (optional) +// given set (typically, RBM_ALLINT or RBM_ALLFLOAT). +// +// Parameters: +// tree - IR node whose internal registers to query +// mask - Mask of registers to count +// +// Returns: +// Count of nodes +// +unsigned NodeInternalRegisters::Count(GenTree* tree, regMaskTP mask) +{ + regMaskTP regs; + return m_table.Lookup(tree, ®s) ? genCountBits(regs & mask) : 0; +} + // CodeGen constructor CodeGenInterface::CodeGenInterface(Compiler* theCompiler) : gcInfo(theCompiler) , regSet(theCompiler, gcInfo) + , internalRegisters(theCompiler) , compiler(theCompiler) , treeLifeUpdater(nullptr) { diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index ef87ccca85870..608c72c22d48d 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -46,6 +46,21 @@ struct RegState CodeGenInterface* getCodeGenerator(Compiler* comp); +class NodeInternalRegisters +{ + typedef JitHashTable, regMaskTP> NodeInternalRegistersTable; + NodeInternalRegistersTable m_table; + +public: + NodeInternalRegisters(Compiler* comp); + + void Add(GenTree* tree, regMaskTP reg); + regNumber Extract(GenTree* tree, regMaskTP mask = static_cast(-1)); + regNumber GetSingle(GenTree* tree, regMaskTP mask = static_cast(-1)); + regMaskTP GetAll(GenTree* tree); + unsigned Count(GenTree* tree, regMaskTP mask = static_cast(-1)); +}; + class CodeGenInterface { friend class emitter; @@ -122,9 +137,10 @@ class CodeGenInterface GCInfo gcInfo; - RegSet regSet; - RegState intRegState; - RegState floatRegState; + RegSet regSet; + RegState intRegState; + RegState floatRegState; + NodeInternalRegisters internalRegisters; protected: Compiler* compiler; diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index d099fe192fc38..786f40c2f4ca7 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -1905,7 +1905,7 @@ void CodeGen::genSetBlockSize(GenTreeBlk* blkNode, regNumber sizeReg) { if (sizeReg != REG_NA) { - assert((blkNode->gtRsvdRegs & genRegMask(sizeReg)) != 0); + assert((internalRegisters.GetAll(blkNode) & genRegMask(sizeReg)) != 0); // This can go via helper which takes the size as a native uint. instGen_Set_Reg_To_Imm(EA_PTRSIZE, sizeReg, blkNode->Size()); } diff --git a/src/coreclr/jit/codegenloongarch64.cpp b/src/coreclr/jit/codegenloongarch64.cpp index 1bd29a432ce17..6fceb11807ed3 100644 --- a/src/coreclr/jit/codegenloongarch64.cpp +++ b/src/coreclr/jit/codegenloongarch64.cpp @@ -1513,7 +1513,7 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types targetType, GenTre else { // Get a temp integer register to compute long address. - // regNumber addrReg = tree->GetSingleTempReg(); + // regNumber addrReg = internalRegisters.GetSingle(tree); // We must load the FP constant from the constant pool // Emit a data section constant for the float or double constant. @@ -1962,12 +1962,12 @@ void CodeGen::genLclHeap(GenTree* tree) // since we don't need any internal registers. if (compiler->info.compInitMem) { - assert(tree->AvailableTempRegCount() == 0); + assert(internalRegisters.Count(tree) == 0); regCnt = targetReg; } else { - regCnt = tree->ExtractTempReg(); + regCnt = internalRegisters.Extract(tree); if (regCnt != targetReg) { emit->emitIns_R_R_I(INS_ori, easz, regCnt, targetReg, 0); @@ -2064,12 +2064,12 @@ void CodeGen::genLclHeap(GenTree* tree) assert(regCnt == REG_NA); if (compiler->info.compInitMem) { - assert(tree->AvailableTempRegCount() == 0); + assert(internalRegisters.Count(tree) == 0); regCnt = targetReg; } else { - regCnt = tree->ExtractTempReg(); + regCnt = internalRegisters.Extract(tree); } instGen_Set_Reg_To_Imm(((unsigned int)amount == amount) ? EA_4BYTE : EA_8BYTE, regCnt, amount); } @@ -2134,7 +2134,7 @@ void CodeGen::genLclHeap(GenTree* tree) // // Setup the regTmp - regNumber regTmp = tree->GetSingleTempReg(); + regNumber regTmp = internalRegisters.GetSingle(tree); assert(regCnt != REG_R21); emit->emitIns_R_R_R(INS_sltu, EA_PTRSIZE, REG_R21, REG_SPBASE, regCnt); @@ -2585,7 +2585,7 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) unsigned slots = layout->GetSlotCount(); // Temp register(s) used to perform the sequence of loads and stores. - regNumber tmpReg = cpObjNode->ExtractTempReg(); + regNumber tmpReg = internalRegisters.Extract(cpObjNode); regNumber tmpReg2 = REG_NA; assert(genIsValidIntReg(tmpReg)); @@ -2594,7 +2594,7 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) if (slots > 1) { - tmpReg2 = cpObjNode->GetSingleTempReg(); + tmpReg2 = internalRegisters.GetSingle(cpObjNode); assert(tmpReg2 != tmpReg); assert(genIsValidIntReg(tmpReg2)); assert(tmpReg2 != REG_WRITE_BARRIER_DST_BYREF); @@ -2729,7 +2729,7 @@ void CodeGen::genTableBasedSwitch(GenTree* treeNode) regNumber idxReg = treeNode->AsOp()->gtOp1->GetRegNum(); regNumber baseReg = treeNode->AsOp()->gtOp2->GetRegNum(); - regNumber tmpReg = treeNode->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(treeNode); // load the ip-relative offset (which is relative to start of fgFirstBB) GetEmitter()->emitIns_R_R_I(INS_slli_d, EA_8BYTE, REG_R21, idxReg, 2); @@ -3591,7 +3591,7 @@ void CodeGen::genCkfinite(GenTree* treeNode) emitAttr attr = emitActualTypeSize(treeNode); // Extract exponent into a register. - regNumber intReg = treeNode->GetSingleTempReg(); + regNumber intReg = internalRegisters.GetSingle(treeNode); regNumber fpReg = genConsumeReg(op1); emit->emitIns_R_R(attr == EA_8BYTE ? INS_movfr2gr_d : INS_movfr2gr_s, attr, intReg, fpReg); @@ -5101,7 +5101,7 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) // Setup loReg from the internal registers that we reserved in lower. // - regNumber loReg = treeNode->ExtractTempReg(); + regNumber loReg = internalRegisters.Extract(treeNode); regNumber addrReg = REG_NA; GenTreeLclVarCommon* varNode = nullptr; @@ -5382,7 +5382,7 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode) assert(source->OperGet() == GT_BLK); assert(varTypeIsStruct(targetType)); - regNumber baseReg = treeNode->ExtractTempReg(); + regNumber baseReg = internalRegisters.Extract(treeNode); regNumber addrReg = REG_NA; GenTreeLclVarCommon* varNode = nullptr; @@ -6019,7 +6019,7 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* cpBlkNode) assert(srcOffset < INT32_MAX - static_cast(size)); assert(dstOffset < INT32_MAX - static_cast(size)); - regNumber tempReg = cpBlkNode->ExtractTempReg(RBM_ALLINT); + regNumber tempReg = internalRegisters.Extract(cpBlkNode, RBM_ALLINT); if (size >= 2 * REGSIZE_BYTES) { @@ -6148,7 +6148,7 @@ void CodeGen::genCodeForInitBlkLoop(GenTreeBlk* initBlkNode) // Extend liveness of dstReg in case if it gets killed by the store. gcInfo.gcMarkRegPtrVal(dstReg, dstNode->TypeGet()); - const regNumber offsetReg = initBlkNode->GetSingleTempReg(); + const regNumber offsetReg = internalRegisters.GetSingle(initBlkNode); instGen_Set_Reg_To_Imm(EA_PTRSIZE, offsetReg, size - TARGET_POINTER_SIZE); // loop begin: @@ -6247,7 +6247,7 @@ void CodeGen::genCall(GenTreeCall* call) (call->IsVirtualStubRelativeIndir() && (call->gtEntryPoint.accessType == IAT_VALUE))); assert(call->gtControlExpr == nullptr); - regNumber tmpReg = call->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(call); // Register where we save call address in should not be overridden by epilog. assert((genRegMask(tmpReg) & (RBM_INT_CALLEE_TRASH & ~RBM_RA)) == genRegMask(tmpReg)); @@ -6255,7 +6255,7 @@ void CodeGen::genCall(GenTreeCall* call) call->IsVirtualStubRelativeIndir() ? compiler->virtualStubParamInfo->GetReg() : REG_R2R_INDIRECT_PARAM; GetEmitter()->emitIns_R_R_I(ins_Load(TYP_I_IMPL), emitActualTypeSize(TYP_I_IMPL), tmpReg, callAddrReg, 0); // We will use this again when emitting the jump in genCallInstruction in the epilog - call->gtRsvdRegs |= genRegMask(tmpReg); + internalRegisters.Add(call, genRegMask(tmpReg)); } #endif @@ -6473,7 +6473,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) if (callThroughIndirReg != REG_NA) { assert(call->IsR2ROrVirtualStubRelativeIndir()); - regNumber targetAddrReg = call->GetSingleTempReg(); + regNumber targetAddrReg = internalRegisters.GetSingle(call); // For fast tailcalls we have already loaded the call target when processing the call node. if (!call->IsFastTailCall()) { @@ -7131,7 +7131,7 @@ void CodeGen::genLeaInstruction(GenTreeAddrMode* lea) } else { - regNumber tmpReg = lea->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(lea); noway_assert(tmpReg != index->GetRegNum()); noway_assert(tmpReg != memBase->GetRegNum()); @@ -7168,7 +7168,7 @@ void CodeGen::genLeaInstruction(GenTreeAddrMode* lea) else { // We require a tmpReg to hold the offset - regNumber tmpReg = lea->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(lea); // First load tmpReg with the large offset constant emit->emitIns_I_la(EA_PTRSIZE, tmpReg, offset); diff --git a/src/coreclr/jit/codegenriscv64.cpp b/src/coreclr/jit/codegenriscv64.cpp index 0fa8bdb5874f4..84a2663dcbcce 100644 --- a/src/coreclr/jit/codegenriscv64.cpp +++ b/src/coreclr/jit/codegenriscv64.cpp @@ -1536,7 +1536,7 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types targetType, GenTre else { // Get a temp integer register to compute long address. - // regNumber addrReg = tree->GetSingleTempReg(); + // regNumber addrReg = internalRegisters.GetSingle(tree); // We must load the FP constant from the constant pool // Emit a data section constant for the float or double constant. @@ -1616,7 +1616,7 @@ void CodeGen::genCodeForMulHi(GenTreeOp* treeNode) assert(EA_SIZE(attr) == EA_4BYTE); if (isUnsigned) { - regNumber tempReg = treeNode->GetSingleTempReg(); + regNumber tempReg = internalRegisters.GetSingle(treeNode); emit->emitIns_R_R_I(INS_slli, EA_8BYTE, tempReg, op1->GetRegNum(), 32); emit->emitIns_R_R_I(INS_slli, EA_8BYTE, targetReg, op2->GetRegNum(), 32); emit->emitIns_R_R_R(INS_mulhu, EA_8BYTE, targetReg, tempReg, targetReg); @@ -1982,7 +1982,7 @@ void CodeGen::genLclHeap(GenTree* tree) } else { - regCnt = tree->ExtractTempReg(); + regCnt = internalRegisters.Extract(tree); if (regCnt != targetReg) { emit->emitIns_R_R_I(INS_ori, easz, regCnt, targetReg, 0); @@ -2012,7 +2012,7 @@ void CodeGen::genLclHeap(GenTree* tree) unsigned outgoingArgSpaceAligned = roundUp(compiler->lvaOutgoingArgSpaceSize, STACK_ALIGN); // assert((compiler->lvaOutgoingArgSpaceSize % STACK_ALIGN) == 0); // This must be true for the stack to remain // // aligned - tempReg = tree->ExtractTempReg(); + tempReg = internalRegisters.Extract(tree); genInstrWithConstant(INS_addi, EA_PTRSIZE, REG_SPBASE, REG_SPBASE, outgoingArgSpaceAligned, tempReg); stackAdjustment += outgoingArgSpaceAligned; } @@ -2067,7 +2067,7 @@ void CodeGen::genLclHeap(GenTree* tree) else { if (tempReg == REG_NA) - tempReg = tree->ExtractTempReg(); + tempReg = internalRegisters.Extract(tree); emit->emitLoadImmediate(EA_PTRSIZE, tempReg, amount); emit->emitIns_R_R_R(INS_sub, EA_PTRSIZE, REG_SPBASE, REG_SPBASE, tempReg); } @@ -2085,7 +2085,7 @@ void CodeGen::genLclHeap(GenTree* tree) } else { - regCnt = tree->ExtractTempReg(); + regCnt = internalRegisters.Extract(tree); } instGen_Set_Reg_To_Imm(((unsigned int)amount == amount) ? EA_4BYTE : EA_8BYTE, regCnt, amount); } @@ -2152,9 +2152,9 @@ void CodeGen::genLclHeap(GenTree* tree) // if (tempReg == REG_NA) - tempReg = tree->ExtractTempReg(); + tempReg = internalRegisters.Extract(tree); - regNumber rPageSize = tree->GetSingleTempReg(); + regNumber rPageSize = internalRegisters.GetSingle(tree); assert(regCnt != tempReg); emit->emitIns_R_R_R(INS_sltu, EA_PTRSIZE, tempReg, REG_SPBASE, regCnt); @@ -2359,7 +2359,7 @@ void CodeGen::genCodeForDivMod(GenTreeOp* tree) ssize_t intConst = (int)(divisorOp->AsIntCon()->gtIconVal); if (!emitter::isGeneralRegister(divisorReg)) { - tempReg = tree->GetSingleTempReg(); + tempReg = internalRegisters.GetSingle(tree); divisorReg = tempReg; } emit->emitLoadImmediate(EA_PTRSIZE, divisorReg, intConst); @@ -2382,7 +2382,7 @@ void CodeGen::genCodeForDivMod(GenTreeOp* tree) if ((exceptions & ExceptionSetFlags::ArithmeticException) != ExceptionSetFlags::None) { if (tempReg == REG_NA) - tempReg = tree->GetSingleTempReg(); + tempReg = internalRegisters.GetSingle(tree); // Check if the divisor is not -1 branch to 'sdivLabel' emit->emitIns_R_R_I(INS_addi, EA_PTRSIZE, tempReg, REG_ZERO, -1); @@ -2608,7 +2608,7 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) unsigned slots = layout->GetSlotCount(); // Temp register(s) used to perform the sequence of loads and stores. - regNumber tmpReg = cpObjNode->ExtractTempReg(); + regNumber tmpReg = internalRegisters.Extract(cpObjNode); regNumber tmpReg2 = REG_NA; assert(genIsValidIntReg(tmpReg)); @@ -2617,7 +2617,7 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) if (slots > 1) { - tmpReg2 = cpObjNode->GetSingleTempReg(); + tmpReg2 = internalRegisters.GetSingle(cpObjNode); assert(tmpReg2 != tmpReg); assert(genIsValidIntReg(tmpReg2)); assert(tmpReg2 != REG_WRITE_BARRIER_DST_BYREF); @@ -2752,7 +2752,7 @@ void CodeGen::genTableBasedSwitch(GenTree* treeNode) regNumber idxReg = treeNode->AsOp()->gtOp1->GetRegNum(); regNumber baseReg = treeNode->AsOp()->gtOp2->GetRegNum(); - regNumber tmpReg = treeNode->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(treeNode); // load the ip-relative offset (which is relative to start of fgFirstBB) GetEmitter()->emitIns_R_R_I(INS_slli, EA_8BYTE, tmpReg, idxReg, 2); @@ -2852,7 +2852,7 @@ void CodeGen::genCodeForCmpXchg(GenTreeCmpXchg* treeNode) regNumber loc = locOp->GetRegNum(); regNumber val = valOp->GetRegNum(); regNumber comparand = comparandOp->GetRegNum(); - regNumber storeErr = treeNode->ExtractTempReg(RBM_ALLINT); + regNumber storeErr = internalRegisters.Extract(treeNode, RBM_ALLINT); // Register allocator should have extended the lifetimes of all input and internal registers // They should all be different @@ -3559,7 +3559,7 @@ void CodeGen::genFloatToIntCast(GenTree* treeNode) genConsumeOperands(treeNode->AsOp()); - regNumber tmpReg = treeNode->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(treeNode); assert(tmpReg != treeNode->GetRegNum()); assert(tmpReg != op1->GetRegNum()); @@ -3606,7 +3606,7 @@ void CodeGen::genCkfinite(GenTree* treeNode) emitAttr attr = emitActualTypeSize(treeNode); // Extract exponent into a register. - regNumber intReg = treeNode->GetSingleTempReg(); + regNumber intReg = internalRegisters.GetSingle(treeNode); regNumber fpReg = genConsumeReg(op1); emit->emitIns_R_R(attr == EA_4BYTE ? INS_fclass_s : INS_fclass_d, attr, intReg, fpReg); @@ -3671,7 +3671,7 @@ void CodeGen::genCodeForCompare(GenTreeOp* tree) } else if (tree->OperIs(GT_EQ)) { - regNumber tempReg = tree->GetSingleTempReg(); + regNumber tempReg = internalRegisters.GetSingle(tree); skipLabel = genCreateTempLabel(); emit->emitIns_R_R(cmpSize == EA_4BYTE ? INS_fclass_s : INS_fclass_d, cmpSize, targetReg, regOp1); emit->emitIns_R_R(cmpSize == EA_4BYTE ? INS_fclass_s : INS_fclass_d, cmpSize, tempReg, regOp2); @@ -3716,7 +3716,7 @@ void CodeGen::genCodeForCompare(GenTreeOp* tree) } else if (tree->OperIs(GT_NE)) { - regNumber tempReg = tree->GetSingleTempReg(); + regNumber tempReg = internalRegisters.GetSingle(tree); emit->emitIns_R_R(cmpSize == EA_4BYTE ? INS_fclass_s : INS_fclass_d, cmpSize, targetReg, regOp1); emit->emitIns_R_R(cmpSize == EA_4BYTE ? INS_fclass_s : INS_fclass_d, cmpSize, tempReg, regOp2); emit->emitIns_R_R_R(INS_or, EA_8BYTE, tempReg, targetReg, tempReg); @@ -3755,7 +3755,7 @@ void CodeGen::genCodeForCompare(GenTreeOp* tree) { imm = static_cast(imm); - regNumber tmpRegOp1 = tree->GetSingleTempReg(); + regNumber tmpRegOp1 = internalRegisters.GetSingle(tree); assert(regOp1 != tmpRegOp1); emit->emitIns_R_R_I(INS_slli, EA_8BYTE, tmpRegOp1, regOp1, 32); @@ -3884,7 +3884,7 @@ void CodeGen::genCodeForCompare(GenTreeOp* tree) if (cmpSize == EA_4BYTE) { regNumber tmpRegOp1 = REG_RA; - regNumber tmpRegOp2 = tree->GetSingleTempReg(); + regNumber tmpRegOp2 = internalRegisters.GetSingle(tree); assert(regOp1 != tmpRegOp2); assert(regOp2 != tmpRegOp2); @@ -5280,7 +5280,7 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) // Setup loReg from the internal registers that we reserved in lower. // - regNumber loReg = treeNode->ExtractTempReg(); + regNumber loReg = internalRegisters.Extract(treeNode); GenTreeLclVarCommon* srcLclNode = nullptr; regNumber addrReg = REG_NA; @@ -5523,7 +5523,7 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode) regNumber allocatedValueReg = REG_NA; if (treeNode->gtNumRegs == 1) { - allocatedValueReg = treeNode->ExtractTempReg(); + allocatedValueReg = internalRegisters.Extract(treeNode); } // Pick a register to store intermediate values in for the to-stack @@ -5670,13 +5670,13 @@ void CodeGen::genRangeCheck(GenTree* oper) if (genActualType(length) == TYP_INT) { - regNumber tempReg = oper->ExtractTempReg(); + regNumber tempReg = internalRegisters.Extract(oper); GetEmitter()->emitIns_R_R_I(INS_addiw, EA_4BYTE, tempReg, lengthReg, 0); // sign-extend lengthReg = tempReg; } if (genActualType(index) == TYP_INT) { - regNumber tempReg = oper->GetSingleTempReg(); + regNumber tempReg = internalRegisters.GetSingle(oper); GetEmitter()->emitIns_R_R_I(INS_addiw, EA_4BYTE, tempReg, indexReg, 0); // sign-extend indexReg = tempReg; } @@ -5759,7 +5759,7 @@ void CodeGen::genCodeForShift(GenTree* tree) if (tree->OperIs(GT_ROR, GT_ROL)) { - regNumber tempReg = tree->GetSingleTempReg(); + regNumber tempReg = internalRegisters.GetSingle(tree); unsigned immWidth = emitter::getBitWidth(size); // For RISCV64, immWidth will be set to 32 or 64 if (!shiftBy->IsCnsIntOrI()) { @@ -5949,7 +5949,7 @@ void CodeGen::genCodeForIndexAddr(GenTreeIndexAddr* node) // The index is never contained, even if it is a constant. assert(index->isUsedFromReg()); - regNumber tempReg = node->GetSingleTempReg(); + regNumber tempReg = internalRegisters.GetSingle(node); // Generate the bounds check if necessary. if (node->IsBoundsChecked()) @@ -6156,7 +6156,7 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* cpBlkNode) assert(srcOffset < INT32_MAX - static_cast(size)); assert(dstOffset < INT32_MAX - static_cast(size)); - regNumber tempReg = cpBlkNode->ExtractTempReg(RBM_ALLINT); + regNumber tempReg = internalRegisters.Extract(cpBlkNode, RBM_ALLINT); if (size >= 2 * REGSIZE_BYTES) { @@ -6285,7 +6285,7 @@ void CodeGen::genCodeForInitBlkLoop(GenTreeBlk* initBlkNode) // Extend liveness of dstReg in case if it gets killed by the store. gcInfo.gcMarkRegPtrVal(dstReg, dstNode->TypeGet()); - const regNumber tempReg = initBlkNode->GetSingleTempReg(); + const regNumber tempReg = internalRegisters.GetSingle(initBlkNode); instGen_Set_Reg_To_Imm(EA_PTRSIZE, tempReg, size - TARGET_POINTER_SIZE); // tempReg = dstReg + tempReg (a new interior pointer, but in a nongc region) @@ -6391,7 +6391,7 @@ void CodeGen::genCall(GenTreeCall* call) (call->IsVirtualStubRelativeIndir() && (call->gtEntryPoint.accessType == IAT_VALUE))); assert(call->gtControlExpr == nullptr); - regNumber tmpReg = call->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(call); // Register where we save call address in should not be overridden by epilog. assert((genRegMask(tmpReg) & (RBM_INT_CALLEE_TRASH & ~RBM_RA)) == genRegMask(tmpReg)); @@ -6399,7 +6399,7 @@ void CodeGen::genCall(GenTreeCall* call) call->IsVirtualStubRelativeIndir() ? compiler->virtualStubParamInfo->GetReg() : REG_R2R_INDIRECT_PARAM; GetEmitter()->emitIns_R_R_I(ins_Load(TYP_I_IMPL), emitActualTypeSize(TYP_I_IMPL), tmpReg, callAddrReg, 0); // We will use this again when emitting the jump in genCallInstruction in the epilog - call->gtRsvdRegs |= genRegMask(tmpReg); + internalRegisters.Add(call, genRegMask(tmpReg)); } #endif @@ -6617,7 +6617,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) if (callThroughIndirReg != REG_NA) { assert(call->IsR2ROrVirtualStubRelativeIndir()); - regNumber targetAddrReg = call->GetSingleTempReg(); + regNumber targetAddrReg = internalRegisters.GetSingle(call); // For fast tailcalls we have already loaded the call target when processing the call node. if (!call->IsFastTailCall()) { @@ -6885,7 +6885,7 @@ void CodeGen::genIntCastOverflowCheck(GenTreeCast* cast, const GenIntCastDesc& d case GenIntCastDesc::CHECK_UINT_RANGE: { - regNumber tempReg = cast->GetSingleTempReg(); + regNumber tempReg = internalRegisters.GetSingle(cast); // We need to check if the value is not greater than 0xFFFFFFFF // if the upper 32 bits are zero. ssize_t imm = -1; @@ -6899,7 +6899,7 @@ void CodeGen::genIntCastOverflowCheck(GenTreeCast* cast, const GenIntCastDesc& d case GenIntCastDesc::CHECK_POSITIVE_INT_RANGE: { - regNumber tempReg = cast->GetSingleTempReg(); + regNumber tempReg = internalRegisters.GetSingle(cast); // We need to check if the value is not greater than 0x7FFFFFFF // if the upper 33 bits are zero. // instGen_Set_Reg_To_Imm(EA_8BYTE, tempReg, 0xFFFFFFFF80000000LL); @@ -6915,7 +6915,7 @@ void CodeGen::genIntCastOverflowCheck(GenTreeCast* cast, const GenIntCastDesc& d case GenIntCastDesc::CHECK_INT_RANGE: { - const regNumber tempReg = cast->GetSingleTempReg(); + const regNumber tempReg = internalRegisters.GetSingle(cast); assert(tempReg != reg); GetEmitter()->emitLoadImmediate(EA_8BYTE, tempReg, INT32_MAX); genJumpToThrowHlpBlk_la(SCK_OVERFLOW, INS_blt, tempReg, nullptr, reg); @@ -6930,7 +6930,7 @@ void CodeGen::genIntCastOverflowCheck(GenTreeCast* cast, const GenIntCastDesc& d assert(desc.CheckKind() == GenIntCastDesc::CHECK_SMALL_INT_RANGE); const int castMaxValue = desc.CheckSmallIntMax(); const int castMinValue = desc.CheckSmallIntMin(); - const regNumber tempReg = cast->GetSingleTempReg(); + const regNumber tempReg = internalRegisters.GetSingle(cast); instruction ins; if (castMaxValue > 2047) @@ -7229,7 +7229,7 @@ void CodeGen::genLeaInstruction(GenTreeAddrMode* lea) assert(isPow2(lea->gtScale)); BitScanForward(&scale, lea->gtScale); assert(scale <= 4); - regNumber scaleTempReg = scale ? lea->ExtractTempReg() : REG_NA; + regNumber scaleTempReg = scale ? internalRegisters.Extract(lea) : REG_NA; if (offset == 0) { @@ -7250,7 +7250,7 @@ void CodeGen::genLeaInstruction(GenTreeAddrMode* lea) } else { - regNumber tmpReg = lea->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(lea); noway_assert(tmpReg != index->GetRegNum()); noway_assert(tmpReg != memBase->GetRegNum()); @@ -7287,7 +7287,7 @@ void CodeGen::genLeaInstruction(GenTreeAddrMode* lea) else { // We require a tmpReg to hold the offset - regNumber tmpReg = lea->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(lea); // First load tmpReg with the large offset constant emit->emitLoadImmediate(EA_PTRSIZE, tmpReg, offset); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 4901ee9e3aa08..a2a17e735c903 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -914,7 +914,7 @@ void CodeGen::genCodeForLongUMod(GenTreeOp* node) // xor edx, edx // div divisor->GetRegNum() // mov eax, temp - const regNumber tempReg = node->GetSingleTempReg(); + const regNumber tempReg = internalRegisters.GetSingle(node); inst_Mov(TYP_INT, tempReg, REG_EAX, /* canSkip */ false); inst_Mov(TYP_INT, REG_EAX, REG_EDX, /* canSkip */ false); instGen_Set_Reg_To_Zero(EA_PTRSIZE, REG_EDX); @@ -1761,7 +1761,7 @@ void CodeGen::genCodeForReturnTrap(GenTreeOp* tree) inst_JMP(EJ_je, skipLabel); // emit the call to the EE-helper that stops for GC (or other reasons) - regNumber tmpReg = tree->GetSingleTempReg(RBM_ALLINT); + regNumber tmpReg = internalRegisters.GetSingle(tree, RBM_ALLINT); assert(genIsValidIntReg(tmpReg)); genEmitHelperCall(CORINFO_HELP_STOP_FOR_GC, 0, EA_UNKNOWN, tmpReg); @@ -2334,7 +2334,7 @@ void CodeGen::genMultiRegStoreToSIMDLocal(GenTreeLclVar* lclNode) } else { - regNumber tempXmm = lclNode->GetSingleTempReg(); + regNumber tempXmm = internalRegisters.GetSingle(lclNode); assert(tempXmm != targetReg); inst_Mov(TYP_FLOAT, tempXmm, reg1, /* canSkip */ false); GetEmitter()->emitIns_SIMD_R_R_R(INS_punpckldq, size, targetReg, targetReg, tempXmm); @@ -2675,7 +2675,7 @@ void CodeGen::genCodeForMemmove(GenTreeBlk* tree) if ((size >= simdSize) && (simdSize > 0)) { // Number of SIMD regs needed to save the whole src to regs. - unsigned numberOfSimdRegs = tree->AvailableTempRegCount(RBM_ALLFLOAT); + unsigned numberOfSimdRegs = internalRegisters.Count(tree, RBM_ALLFLOAT); // Lowering takes care to only introduce this node such that we will always have enough // temporary SIMD registers to fully load the source and avoid any potential issues with overlap. @@ -2685,7 +2685,7 @@ void CodeGen::genCodeForMemmove(GenTreeBlk* tree) regNumber tempRegs[LinearScan::MaxInternalCount] = {}; for (unsigned i = 0; i < numberOfSimdRegs; i++) { - tempRegs[i] = tree->ExtractTempReg(RBM_ALLFLOAT); + tempRegs[i] = internalRegisters.Extract(tree, RBM_ALLFLOAT); } auto emitSimdLoadStore = [&](bool load) { @@ -2770,15 +2770,15 @@ void CodeGen::genCodeForMemmove(GenTreeBlk* tree) unsigned loadStoreSize = 1 << BitOperations::Log2(size); if (loadStoreSize == size) { - regNumber tmpReg = tree->GetSingleTempReg(RBM_ALLINT); + regNumber tmpReg = internalRegisters.GetSingle(tree, RBM_ALLINT); emitScalarLoadStore(/* load */ true, loadStoreSize, tmpReg, 0); emitScalarLoadStore(/* load */ false, loadStoreSize, tmpReg, 0); } else { - assert(tree->AvailableTempRegCount() == 2); - regNumber tmpReg1 = tree->ExtractTempReg(RBM_ALLINT); - regNumber tmpReg2 = tree->ExtractTempReg(RBM_ALLINT); + assert(internalRegisters.Count(tree) == 2); + regNumber tmpReg1 = internalRegisters.Extract(tree, RBM_ALLINT); + regNumber tmpReg2 = internalRegisters.Extract(tree, RBM_ALLINT); emitScalarLoadStore(/* load */ true, loadStoreSize, tmpReg1, 0); emitScalarLoadStore(/* load */ true, loadStoreSize, tmpReg2, size - loadStoreSize); emitScalarLoadStore(/* load */ false, loadStoreSize, tmpReg1, 0); @@ -2855,12 +2855,12 @@ void CodeGen::genLclHeap(GenTree* tree) // since we don't need any internal registers. if (compiler->info.compInitMem) { - assert(tree->AvailableTempRegCount() == 0); + assert(internalRegisters.Count(tree) == 0); regCnt = targetReg; } else { - regCnt = tree->GetSingleTempReg(); + regCnt = internalRegisters.GetSingle(tree); // Above, we put the size in targetReg. Now, copy it to our new temp register if necessary. inst_Mov(size->TypeGet(), regCnt, targetReg, /* canSkip */ true); @@ -2954,7 +2954,7 @@ void CodeGen::genLclHeap(GenTree* tree) // via BLK explicitly, so just bump the stack pointer. if ((amount >= compiler->eeGetPageSize()) || (TARGET_POINTER_SIZE == 4)) { - regCnt = tree->GetSingleTempReg(); + regCnt = internalRegisters.GetSingle(tree); instGen_Set_Reg_To_Imm(EA_PTRSIZE, regCnt, -(ssize_t)amount); genStackPointerDynamicAdjustmentWithProbe(regCnt); // lastTouchDelta is dynamic, and can be up to a page. So if we have outgoing arg space, @@ -2971,7 +2971,7 @@ void CodeGen::genLclHeap(GenTree* tree) } // We should not have any temp registers at this point. - assert(tree->AvailableTempRegCount() == 0); + assert(internalRegisters.Count(tree) == 0); if (compiler->info.compInitMem) { @@ -3234,7 +3234,7 @@ void CodeGen::genCodeForInitBlkUnroll(GenTreeBlk* node) #ifdef FEATURE_SIMD if (willUseSimdMov) { - regNumber srcXmmReg = node->GetSingleTempReg(RBM_ALLFLOAT); + regNumber srcXmmReg = internalRegisters.GetSingle(node, RBM_ALLFLOAT); unsigned regSize = compiler->roundDownSIMDSize(size); var_types loadType = compiler->getSIMDTypeForSize(regSize); simd_t vecCon; @@ -3392,7 +3392,7 @@ void CodeGen::genCodeForInitBlkLoop(GenTreeBlk* initBlkNode) // Extend liveness of dstReg in case if it gets killed by the store. gcInfo.gcMarkRegPtrVal(dstReg, dstNode->TypeGet()); - const regNumber offsetReg = initBlkNode->GetSingleTempReg(); + const regNumber offsetReg = internalRegisters.GetSingle(initBlkNode); instGen_Set_Reg_To_Imm(EA_PTRSIZE, offsetReg, size - TARGET_POINTER_SIZE); BasicBlock* loop = genCreateTempLabel(); @@ -3531,7 +3531,7 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) if ((size >= regSize) && (regSize > 0)) { - regNumber tempReg = node->GetSingleTempReg(RBM_ALLFLOAT); + regNumber tempReg = internalRegisters.GetSingle(node, RBM_ALLFLOAT); instruction simdMov = simdUnalignedMovIns(); @@ -3593,7 +3593,7 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) // Fill the remainder with normal loads/stores if (size > 0) { - regNumber tempReg = node->GetSingleTempReg(RBM_ALLINT); + regNumber tempReg = internalRegisters.GetSingle(node, RBM_ALLINT); #ifdef TARGET_AMD64 unsigned regSize = REGSIZE_BYTES; @@ -3878,11 +3878,11 @@ void CodeGen::genStructPutArgUnroll(GenTreePutArgStk* putArgNode) if (loadSize >= XMM_REGSIZE_BYTES) #endif { - xmmTmpReg = putArgNode->GetSingleTempReg(RBM_ALLFLOAT); + xmmTmpReg = internalRegisters.GetSingle(putArgNode, RBM_ALLFLOAT); } if ((loadSize % XMM_REGSIZE_BYTES) != 0) { - intTmpReg = putArgNode->GetSingleTempReg(RBM_ALLINT); + intTmpReg = internalRegisters.GetSingle(putArgNode, RBM_ALLINT); } #ifdef TARGET_X86 @@ -3934,7 +3934,7 @@ void CodeGen::genStructPutArgRepMovs(GenTreePutArgStk* putArgNode) // Make sure we got the arguments of the cpblk operation in the right registers, and that // 'src' is contained as expected. - assert(putArgNode->gtRsvdRegs == (RBM_RDI | RBM_RCX | RBM_RSI)); + assert(internalRegisters.GetAll(putArgNode) == (RBM_RDI | RBM_RCX | RBM_RSI)); assert(src->isContained()); genConsumePutStructArgStk(putArgNode, REG_RDI, REG_RSI, REG_RCX); @@ -4219,7 +4219,7 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) { // If the destination of the CpObj is on the stack, make sure we allocated // RCX to emit the movsp (alias for movsd or movsq for 32 and 64 bits respectively). - assert((cpObjNode->gtRsvdRegs & RBM_RCX) != 0); + assert((internalRegisters.GetAll(cpObjNode) & RBM_RCX) != 0); GetEmitter()->emitIns_R_I(INS_mov, EA_4BYTE, REG_RCX, slots); instGen(INS_r_movsp); @@ -4269,7 +4269,7 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) { // Otherwise, we can save code-size and improve CQ by emitting // rep movsp (alias for movsd/movsq for x86/x64) - assert((cpObjNode->gtRsvdRegs & RBM_RCX) != 0); + assert((internalRegisters.GetAll(cpObjNode) & RBM_RCX) != 0); GetEmitter()->emitIns_R_I(INS_mov, EA_4BYTE, REG_RCX, nonGcSlotCount); instGen(INS_r_movsp); @@ -4300,7 +4300,7 @@ void CodeGen::genTableBasedSwitch(GenTree* treeNode) regNumber idxReg = treeNode->AsOp()->gtOp1->GetRegNum(); regNumber baseReg = treeNode->AsOp()->gtOp2->GetRegNum(); - regNumber tmpReg = treeNode->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(treeNode); // load the ip-relative offset (which is relative to start of fgFirstBB) GetEmitter()->emitIns_R_ARX(INS_mov, EA_4BYTE, baseReg, baseReg, idxReg, 4, 0); @@ -4427,7 +4427,7 @@ void CodeGen::genLockedInstructions(GenTreeOp* node) // Extend liveness of addr gcInfo.gcMarkRegPtrVal(addr->GetRegNum(), addr->TypeGet()); - const regNumber tmpReg = node->GetSingleTempReg(); + const regNumber tmpReg = internalRegisters.GetSingle(node); GetEmitter()->emitIns_R_AR(INS_mov, size, REG_RAX, addr->GetRegNum(), 0); BasicBlock* loop = genCreateTempLabel(); genDefineTempLabel(loop); @@ -5306,7 +5306,7 @@ void CodeGen::genCodeForIndexAddr(GenTreeIndexAddr* node) regNumber tmpReg = REG_NA; #ifdef TARGET_64BIT - tmpReg = node->GetSingleTempReg(); + tmpReg = internalRegisters.GetSingle(node); #endif // Generate the bounds check if necessary. @@ -5356,7 +5356,7 @@ void CodeGen::genCodeForIndexAddr(GenTreeIndexAddr* node) // The VM doesn't allow such large array elements but let's be sure. noway_assert(scale <= INT32_MAX); #else // !TARGET_64BIT - tmpReg = node->GetSingleTempReg(); + tmpReg = internalRegisters.GetSingle(node); #endif // !TARGET_64BIT GetEmitter()->emitIns_R_I(emitter::inst3opImulForReg(tmpReg), EA_PTRSIZE, indexReg, @@ -7239,7 +7239,7 @@ void CodeGen::genIntCastOverflowCheck(GenTreeCast* cast, const GenIntCastDesc& d // We need to check if the value is not greater than 0xFFFFFFFF but this value // cannot be encoded in an immediate operand. Use a right shift to test if the // upper 32 bits are zero. This requires a temporary register. - const regNumber tempReg = cast->GetSingleTempReg(); + const regNumber tempReg = internalRegisters.GetSingle(cast); assert(tempReg != reg); GetEmitter()->emitIns_Mov(INS_mov, EA_8BYTE, tempReg, reg, /* canSkip */ false); GetEmitter()->emitIns_R_I(INS_shr_N, EA_8BYTE, tempReg, 32); @@ -7255,7 +7255,7 @@ void CodeGen::genIntCastOverflowCheck(GenTreeCast* cast, const GenIntCastDesc& d case GenIntCastDesc::CHECK_INT_RANGE: { // Emit "if ((long)(int)x != x) goto OVERFLOW" - const regNumber regTmp = cast->GetSingleTempReg(); + const regNumber regTmp = internalRegisters.GetSingle(cast); GetEmitter()->emitIns_Mov(INS_movsxd, EA_8BYTE, regTmp, reg, true); GetEmitter()->emitIns_R_R(INS_cmp, EA_8BYTE, reg, regTmp); genJumpToThrowHlpBlk(EJ_jne, SCK_OVERFLOW); @@ -7666,7 +7666,7 @@ void CodeGen::genCkfinite(GenTree* treeNode) regNumber targetReg = treeNode->GetRegNum(); // Extract exponent into a register. - regNumber tmpReg = treeNode->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(treeNode); genConsumeReg(op1); @@ -8357,17 +8357,17 @@ void CodeGen::genPutArgStkFieldList(GenTreePutArgStk* putArgStk) unsigned prevFieldOffset = currentOffset; regNumber intTmpReg = REG_NA; regNumber simdTmpReg = REG_NA; - if (putArgStk->AvailableTempRegCount() != 0) + if (internalRegisters.Count(putArgStk) != 0) { - regMaskTP rsvdRegs = putArgStk->gtRsvdRegs; + regMaskTP rsvdRegs = internalRegisters.GetAll(putArgStk); if ((rsvdRegs & RBM_ALLINT) != 0) { - intTmpReg = putArgStk->GetSingleTempReg(RBM_ALLINT); + intTmpReg = internalRegisters.GetSingle(putArgStk, RBM_ALLINT); assert(genIsValidIntReg(intTmpReg)); } if ((rsvdRegs & RBM_ALLFLOAT) != 0) { - simdTmpReg = putArgStk->GetSingleTempReg(RBM_ALLFLOAT); + simdTmpReg = internalRegisters.GetSingle(putArgStk, RBM_ALLFLOAT); assert(genIsValidFloatReg(simdTmpReg)); } assert(genCountBits(rsvdRegs) == (unsigned)((intTmpReg == REG_NA) ? 0 : 1) + ((simdTmpReg == REG_NA) ? 0 : 1)); diff --git a/src/coreclr/jit/emitarm.cpp b/src/coreclr/jit/emitarm.cpp index 81331547b4a92..65a6e098b2976 100644 --- a/src/coreclr/jit/emitarm.cpp +++ b/src/coreclr/jit/emitarm.cpp @@ -7869,15 +7869,15 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR var_types type = indir->AsStoreInd()->Data()->TypeGet(); if (type == TYP_FLOAT) { - regNumber tmpReg = indir->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(indir); emitIns_Mov(INS_vmov_f2i, EA_4BYTE, tmpReg, dataReg, /* canSkip */ false); emitInsLoadStoreOp(INS_str, EA_4BYTE, tmpReg, indir, 0); return; } else if (type == TYP_DOUBLE) { - regNumber tmpReg1 = indir->ExtractTempReg(); - regNumber tmpReg2 = indir->GetSingleTempReg(); + regNumber tmpReg1 = codeGen->internalRegisters.Extract(indir); + regNumber tmpReg2 = codeGen->internalRegisters.GetSingle(indir); emitIns_R_R_R(INS_vmov_d2i, EA_8BYTE, tmpReg1, tmpReg2, dataReg); emitInsLoadStoreOp(INS_str, EA_4BYTE, tmpReg1, indir, 0); emitInsLoadStoreOp(INS_str, EA_4BYTE, tmpReg2, indir, 4); @@ -7889,15 +7889,15 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR var_types type = indir->TypeGet(); if (type == TYP_FLOAT) { - regNumber tmpReg = indir->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(indir); emitInsLoadStoreOp(INS_ldr, EA_4BYTE, tmpReg, indir, 0); emitIns_Mov(INS_vmov_i2f, EA_4BYTE, dataReg, tmpReg, /* canSkip */ false); return; } else if (type == TYP_DOUBLE) { - regNumber tmpReg1 = indir->ExtractTempReg(); - regNumber tmpReg2 = indir->GetSingleTempReg(); + regNumber tmpReg1 = codeGen->internalRegisters.Extract(indir); + regNumber tmpReg2 = codeGen->internalRegisters.GetSingle(indir); emitInsLoadStoreOp(INS_ldr, EA_4BYTE, tmpReg1, indir, 0); emitInsLoadStoreOp(INS_ldr, EA_4BYTE, tmpReg2, indir, 4); emitIns_R_R_R(INS_vmov_i2d, EA_8BYTE, dataReg, tmpReg1, tmpReg2); @@ -7940,7 +7940,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR if (offset != 0) { - regNumber tmpReg = indir->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(indir); // If the LEA produces a GCREF or BYREF, we need to be careful to mark any temp register // computed with the base register as a BYREF. @@ -8023,7 +8023,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR else { // We require a tmpReg to hold the offset - regNumber tmpReg = indir->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(indir); // First load/store tmpReg with the large offset constant codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, tmpReg, offset); @@ -8175,7 +8175,7 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, { if (isMulOverflow) { - regNumber extraReg = dst->GetSingleTempReg(); + regNumber extraReg = codeGen->internalRegisters.GetSingle(dst); assert(extraReg != dst->GetRegNum()); if ((dst->gtFlags & GTF_UNSIGNED) != 0) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 181b9706e4161..3c988633e798b 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -14243,7 +14243,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR if (offset != 0) { - regNumber tmpReg = indir->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(indir); emitAttr addType = varTypeIsGC(memBase) ? EA_BYREF : EA_PTRSIZE; @@ -14350,7 +14350,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR else { // We require a tmpReg to hold the offset - regNumber tmpReg = indir->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(indir); // First load/store tmpReg with the large offset constant codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, tmpReg, offset); @@ -14490,7 +14490,7 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, { if (isMulOverflow) { - regNumber extraReg = dst->GetSingleTempReg(); + regNumber extraReg = codeGen->internalRegisters.GetSingle(dst); assert(extraReg != dst->GetRegNum()); if ((dst->gtFlags & GTF_UNSIGNED) != 0) @@ -16996,7 +16996,7 @@ void emitter::emitStoreSimd12ToLclOffset(unsigned varNum, unsigned offset, regNu emitIns_S_R(INS_str, EA_8BYTE, dataReg, varNum, offset); // Extract upper 4-bytes from data - regNumber tmpReg = tmpRegProvider->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(tmpRegProvider); emitIns_R_R_I(INS_mov, EA_4BYTE, tmpReg, dataReg, 2); // 4-byte write diff --git a/src/coreclr/jit/emitloongarch64.cpp b/src/coreclr/jit/emitloongarch64.cpp index 63d4e9a975b9b..43281e536e8b3 100644 --- a/src/coreclr/jit/emitloongarch64.cpp +++ b/src/coreclr/jit/emitloongarch64.cpp @@ -4586,7 +4586,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR if (offset != 0) { - regNumber tmpReg = indir->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(indir); if (isValidSimm12(offset)) { @@ -4727,7 +4727,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR else { // We require a tmpReg to hold the offset - regNumber tmpReg = indir->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(indir); // First load/store tmpReg with the large offset constant emitIns_I_la(EA_PTRSIZE, tmpReg, offset); @@ -5058,7 +5058,7 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, if ((dst->gtFlags & GTF_UNSIGNED) == 0) { - saveOperReg2 = dst->GetSingleTempReg(); + saveOperReg2 = codeGen->internalRegisters.GetSingle(dst); assert((saveOperReg2 != REG_RA) && (saveOperReg2 != REG_R21)); assert(REG_RA != regOp1); assert(saveOperReg2 != regOp2); diff --git a/src/coreclr/jit/emitriscv64.cpp b/src/coreclr/jit/emitriscv64.cpp index d78fc0e9d5b29..932c9a1125b01 100644 --- a/src/coreclr/jit/emitriscv64.cpp +++ b/src/coreclr/jit/emitriscv64.cpp @@ -4463,7 +4463,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR if (offset != 0) { - regNumber tmpReg = indir->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(indir); if (isValidSimm12(offset)) { @@ -4496,7 +4496,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR noway_assert(emitInsIsLoad(ins) || (tmpReg != dataReg)); noway_assert(tmpReg != index->GetRegNum()); - regNumber scaleReg = indir->GetSingleTempReg(); + regNumber scaleReg = codeGen->internalRegisters.GetSingle(indir); // Then load/store dataReg from/to [tmpReg + index*scale] emitIns_R_R_I(INS_slli, addType, scaleReg, index->GetRegNum(), lsl); emitIns_R_R_R(INS_add, addType, tmpReg, tmpReg, scaleReg); @@ -4605,7 +4605,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR else { // We require a tmpReg to hold the offset - regNumber tmpReg = indir->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(indir); // First load/store tmpReg with the large offset constant emitLoadImmediate(EA_PTRSIZE, tmpReg, offset); @@ -4771,7 +4771,7 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, assert(ins == INS_addi || ins == INS_addiw || ins == INS_andi || ins == INS_ori || ins == INS_xori); - regNumber tempReg = needCheckOv ? dst->ExtractTempReg() : REG_NA; + regNumber tempReg = needCheckOv ? codeGen->internalRegisters.Extract(dst) : REG_NA; if (needCheckOv) { @@ -4823,7 +4823,7 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, } else { - regNumber tempReg = needCheckOv ? dst->ExtractTempReg() : REG_NA; + regNumber tempReg = needCheckOv ? codeGen->internalRegisters.Extract(dst) : REG_NA; switch (dst->OperGet()) { @@ -4897,7 +4897,7 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, } else { - regNumber tempReg2 = dst->ExtractTempReg(); + regNumber tempReg2 = codeGen->internalRegisters.Extract(dst); assert(tempReg2 != dstReg); assert(tempReg2 != src1Reg); assert(tempReg2 != src2Reg); @@ -5003,7 +5003,7 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, else { tempReg1 = REG_RA; - tempReg2 = dst->ExtractTempReg(); + tempReg2 = codeGen->internalRegisters.Extract(dst); assert(tempReg1 != tempReg2); assert(tempReg1 != saveOperReg1); assert(tempReg2 != saveOperReg2); diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 6bf148cf2d888..35cb3a986ee58 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -5641,7 +5641,7 @@ void emitter::emitStoreSimd12ToLclOffset(unsigned varNum, unsigned offset, regNu } else { - regNumber tmpReg = tmpRegProvider->GetSingleTempReg(); + regNumber tmpReg = codeGen->internalRegisters.GetSingle(tmpRegProvider); assert(isFloatReg(tmpReg)); // Extract upper 4 bytes from data diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 8076662e183e4..d48fc45868ba2 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -9782,7 +9782,6 @@ GenTree* Compiler::gtCloneExpr(GenTree* tree) /* Make sure to copy back fields that may have been initialized */ copy->CopyRawCosts(tree); - copy->gtRsvdRegs = tree->gtRsvdRegs; copy->CopyReg(tree); return copy; } @@ -11644,7 +11643,7 @@ void Compiler::gtDispNode(GenTree* tree, IndentStack* indentStack, _In_ _In_opt_ if (verbose && 0) { printf(" RR="); - dspRegMask(tree->gtRsvdRegs); + dspRegMask(JitTls::GetCompiler()->codeGen->internalRegisters.GetAll(tree)); printf("\n"); } } @@ -27720,75 +27719,14 @@ regMaskTP ReturnTypeDesc::GetABIReturnRegs(CorInfoCallConvExtension callConv) co } //------------------------------------------------------------------------ -// The following functions manage the gtRsvdRegs set of temporary registers -// created by LSRA during code generation. - -//------------------------------------------------------------------------ -// AvailableTempRegCount: return the number of available temporary registers in the (optional) given set -// (typically, RBM_ALLINT or RBM_ALLFLOAT). -// -// Arguments: -// mask - (optional) Check for available temporary registers only in this set. -// -// Return Value: -// Count of available temporary registers in given set. -// -unsigned GenTree::AvailableTempRegCount(regMaskTP mask /* = (regMaskTP)-1 */) const -{ - return genCountBits(gtRsvdRegs & mask); -} - -//------------------------------------------------------------------------ -// GetSingleTempReg: There is expected to be exactly one available temporary register -// in the given mask in the gtRsvdRegs set. Get that register. No future calls to get -// a temporary register are expected. Removes the register from the set, but only in -// DEBUG to avoid doing unnecessary work in non-DEBUG builds. +// GetNum: Get the SSA number for a given field. // -// Arguments: -// mask - (optional) Get an available temporary register only in this set. -// -// Return Value: -// Available temporary register in given mask. -// -regNumber GenTree::GetSingleTempReg(regMaskTP mask /* = (regMaskTP)-1 */) -{ - regMaskTP availableSet = gtRsvdRegs & mask; - assert(genCountBits(availableSet) == 1); - regNumber tempReg = genRegNumFromMask(availableSet); - INDEBUG(gtRsvdRegs &= ~availableSet;) // Remove the register from the set, so it can't be used again. - return tempReg; -} - -//------------------------------------------------------------------------ -// ExtractTempReg: Find the lowest number temporary register from the gtRsvdRegs set -// that is also in the optional given mask (typically, RBM_ALLINT or RBM_ALLFLOAT), -// and return it. Remove this register from the temporary register set, so it won't -// be returned again. -// -// Arguments: -// mask - (optional) Extract an available temporary register only in this set. -// -// Return Value: -// Available temporary register in given mask. -// -regNumber GenTree::ExtractTempReg(regMaskTP mask /* = (regMaskTP)-1 */) -{ - regMaskTP availableSet = gtRsvdRegs & mask; - assert(genCountBits(availableSet) >= 1); - regNumber tempReg = genFirstRegNumFromMask(availableSet); - gtRsvdRegs ^= genRegMask(tempReg); - return tempReg; -} - -//------------------------------------------------------------------------ -// GetNum: Get the SSA number for a given field. -// -// Arguments: -// compiler - The Compiler instance -// index - The field index +// Arguments: +// compiler - The Compiler instance +// index - The field index // -// Return Value: -// The SSA number corresponding to the field at "index". +// Return Value: +// The SSA number corresponding to the field at "index". // unsigned SsaNumInfo::GetNum(Compiler* compiler, unsigned index) const { diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 2f3a2a7b2f573..7dbfef174994e 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -956,12 +956,6 @@ struct GenTree ValueNumPair gtVNPair; - regMaskSmall gtRsvdRegs; // set of fixed trashed registers - - unsigned AvailableTempRegCount(regMaskTP mask = (regMaskTP)-1) const; - regNumber GetSingleTempReg(regMaskTP mask = (regMaskTP)-1); - regNumber ExtractTempReg(regMaskTP mask = (regMaskTP)-1); - void SetVNsFromNode(GenTree* tree) { gtVNPair = tree->gtVNPair; diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index aadd8fcf28e00..f58ca6c6e858d 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -99,7 +99,7 @@ CodeGen::HWIntrinsicImmOpHelper::HWIntrinsicImmOpHelper(CodeGen* codeGen, GenTre // using the same approach as in hwintrinsicxarch.cpp - adding an additional indirection level in form of a // branch table. assert(!HWIntrinsicInfo::GeneratesMultipleIns(intrin->GetHWIntrinsicId())); - branchTargetReg = intrin->GetSingleTempReg(); + branchTargetReg = codeGen->internalRegisters.GetSingle(intrin); } endLabel = codeGen->genCreateTempLabel(); @@ -1206,7 +1206,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) if (intrin.op1->OperIsLocal()) { unsigned varNum = intrin.op1->AsLclVarCommon()->GetLclNum(); - baseReg = node->ExtractTempReg(); + baseReg = internalRegisters.Extract(node); // Load the address of varNum GetEmitter()->emitIns_R_S(INS_lea, EA_PTRSIZE, baseReg, varNum, 0); @@ -1226,7 +1226,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) unsigned simdInitTempVarNum = compiler->lvaSIMDInitTempVarNum; noway_assert(simdInitTempVarNum != BAD_VAR_NUM); - baseReg = node->ExtractTempReg(); + baseReg = internalRegisters.Extract(node); // Load the address of simdInitTempVarNum GetEmitter()->emitIns_R_S(INS_lea, EA_PTRSIZE, baseReg, simdInitTempVarNum, 0); diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index 79e6b497c368a..a4b6748f3db53 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -323,8 +323,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) insOpts newInstOptions = AddEmbRoundingMode(instOptions, i); genHWIntrinsic_R_RM(node, ins, simdSize, targetReg, rmOp, newInstOptions); }; - regNumber baseReg = node->ExtractTempReg(); - regNumber offsReg = node->GetSingleTempReg(); + regNumber baseReg = internalRegisters.Extract(node); + regNumber offsReg = internalRegisters.GetSingle(node); genHWIntrinsicJumpTableFallback(intrinsicId, lastOp->GetRegNum(), baseReg, offsReg, emitSwCase); break; @@ -335,8 +335,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) insOpts newInstOptions = AddEmbRoundingMode(instOptions, i); genHWIntrinsic_R_R_RM(node, ins, simdSize, newInstOptions); }; - regNumber baseReg = node->ExtractTempReg(); - regNumber offsReg = node->GetSingleTempReg(); + regNumber baseReg = internalRegisters.Extract(node); + regNumber offsReg = internalRegisters.GetSingle(node); genHWIntrinsicJumpTableFallback(intrinsicId, lastOp->GetRegNum(), baseReg, offsReg, emitSwCase); break; @@ -524,8 +524,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) // Reflection. However, it // can also occur if the consumer calls it directly and just doesn't pass a // constant value. - regNumber baseReg = node->ExtractTempReg(); - regNumber offsReg = node->GetSingleTempReg(); + regNumber baseReg = internalRegisters.Extract(node); + regNumber offsReg = internalRegisters.GetSingle(node); genHWIntrinsicJumpTableFallback(intrinsicId, op2Reg, baseReg, offsReg, emitSwCase); } } @@ -574,8 +574,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) // We emit a fallback case for the scenario when the imm-op is not a constant. This should // normally happen when the intrinsic is called indirectly, such as via Reflection. However, it // can also occur if the consumer calls it directly and just doesn't pass a constant value. - regNumber baseReg = node->ExtractTempReg(); - regNumber offsReg = node->GetSingleTempReg(); + regNumber baseReg = internalRegisters.Extract(node); + regNumber offsReg = internalRegisters.GetSingle(node); genHWIntrinsicJumpTableFallback(intrinsicId, op3Reg, baseReg, offsReg, emitSwCase); } } @@ -670,8 +670,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) // We emit a fallback case for the scenario when the imm-op is not a constant. This should // normally happen when the intrinsic is called indirectly, such as via Reflection. However, it // can also occur if the consumer calls it directly and just doesn't pass a constant value. - regNumber baseReg = node->ExtractTempReg(); - regNumber offsReg = node->GetSingleTempReg(); + regNumber baseReg = internalRegisters.Extract(node); + regNumber offsReg = internalRegisters.GetSingle(node); genHWIntrinsicJumpTableFallback(intrinsicId, op4Reg, baseReg, offsReg, emitSwCase); } } @@ -1373,8 +1373,8 @@ void CodeGen::genNonTableDrivenHWIntrinsicsJumpTableFallback(GenTreeHWIntrinsic* insOpts newInstOptions = AddEmbRoundingMode(instOptions, i); genHWIntrinsic_R_RM(node, ins, attr, targetReg, rmOp, newInstOptions); }; - regNumber baseReg = node->ExtractTempReg(); - regNumber offsReg = node->GetSingleTempReg(); + regNumber baseReg = internalRegisters.Extract(node); + regNumber offsReg = internalRegisters.GetSingle(node); genHWIntrinsicJumpTableFallback(intrinsicId, lastOp->GetRegNum(), baseReg, offsReg, emitSwCase); break; } @@ -1394,8 +1394,8 @@ void CodeGen::genNonTableDrivenHWIntrinsicsJumpTableFallback(GenTreeHWIntrinsic* insOpts newInstOptions = AddEmbRoundingMode(instOptions, i); genHWIntrinsic_R_RM(node, ins, attr, targetReg, rmOp, newInstOptions); }; - regNumber baseReg = node->ExtractTempReg(); - regNumber offsReg = node->GetSingleTempReg(); + regNumber baseReg = internalRegisters.Extract(node); + regNumber offsReg = internalRegisters.GetSingle(node); genHWIntrinsicJumpTableFallback(intrinsicId, lastOp->GetRegNum(), baseReg, offsReg, emitSwCase); break; } @@ -1408,8 +1408,8 @@ void CodeGen::genNonTableDrivenHWIntrinsicsJumpTableFallback(GenTreeHWIntrinsic* insOpts newInstOptions = AddEmbRoundingMode(instOptions, i); genHWIntrinsic_R_R_RM(node, ins, EA_8BYTE, newInstOptions); }; - regNumber baseReg = node->ExtractTempReg(); - regNumber offsReg = node->GetSingleTempReg(); + regNumber baseReg = internalRegisters.Extract(node); + regNumber offsReg = internalRegisters.GetSingle(node); genHWIntrinsicJumpTableFallback(intrinsicId, lastOp->GetRegNum(), baseReg, offsReg, emitSwCase); break; } @@ -1440,8 +1440,8 @@ void CodeGen::genNonTableDrivenHWIntrinsicsJumpTableFallback(GenTreeHWIntrinsic* insOpts newInstOptions = AddEmbRoundingMode(instOptions, i); genHWIntrinsic_R_R_R_RM(ins, attr, targetReg, op1Reg, op2Reg, op3, newInstOptions); }; - regNumber baseReg = node->ExtractTempReg(); - regNumber offsReg = node->GetSingleTempReg(); + regNumber baseReg = internalRegisters.Extract(node); + regNumber offsReg = internalRegisters.GetSingle(node); genHWIntrinsicJumpTableFallback(intrinsicId, lastOp->GetRegNum(), baseReg, offsReg, emitSwCase); break; } @@ -2074,8 +2074,8 @@ void CodeGen::genSSE41Intrinsic(GenTreeHWIntrinsic* node) // We emit a fallback case for the scenario when the imm-op is not a constant. This should // normally happen when the intrinsic is called indirectly, such as via Reflection. However, it // can also occur if the consumer calls it directly and just doesn't pass a constant value. - regNumber baseReg = node->ExtractTempReg(); - regNumber offsReg = node->GetSingleTempReg(); + regNumber baseReg = internalRegisters.Extract(node); + regNumber offsReg = internalRegisters.GetSingle(node); genHWIntrinsicJumpTableFallback(intrinsicId, op2->GetRegNum(), baseReg, offsReg, emitSwCase); } break; @@ -2225,7 +2225,7 @@ void CodeGen::genAvxFamilyIntrinsic(GenTreeHWIntrinsic* node, insOpts instOption regNumber op2Reg = op2->GetRegNum(); regNumber addrBaseReg = REG_NA; regNumber addrIndexReg = REG_NA; - regNumber maskReg = node->ExtractTempReg(RBM_ALLFLOAT); + regNumber maskReg = internalRegisters.Extract(node, RBM_ALLFLOAT); if (numArgs == 5) { @@ -2902,7 +2902,7 @@ void CodeGen::genBMI1OrBMI2Intrinsic(GenTreeHWIntrinsic* node, insOpts instOptio assert(op3Reg != op1Reg); assert(op3Reg != targetReg); assert(op3Reg != REG_EDX); - lowReg = node->GetSingleTempReg(); + lowReg = internalRegisters.GetSingle(node); assert(op3Reg != lowReg); assert(lowReg != targetReg); } diff --git a/src/coreclr/jit/jithashtable.h b/src/coreclr/jit/jithashtable.h index f699c3eee19d2..7f9153b75864a 100644 --- a/src/coreclr/jit/jithashtable.h +++ b/src/coreclr/jit/jithashtable.h @@ -233,7 +233,7 @@ class JitHashTable } //------------------------------------------------------------------------ - // Lookup: Get a pointer to the value associated to the specified key. + // LookupPointer: Get a pointer to the value associated to the specified key. // if any. // // Arguments: @@ -261,6 +261,48 @@ class JitHashTable } } + //------------------------------------------------------------------------ + // LookupPointerOrAdd: Get a pointer to the value associated to the specified key. + // If not present, add it with the specified default value and return a pointer to it. + // + // Arguments: + // k - the key + // defaultValue - Default value to add to the table if the key was not present + // + // Return Value: + // A pointer to the value associated with the specified key. + // + Value* LookupPointerOrAdd(Key k, Value defaultValue) + { + CheckGrowth(); + + assert(m_tableSizeInfo.prime != 0); + + unsigned index = GetIndexForKey(k); + + Node* n = m_table[index]; + while (n != nullptr) + { + if (KeyFuncs::Equals(k, n->m_key)) + { + return &n->m_val; + } + + n = n->m_next; + } + + n = new (m_alloc) Node(m_table[index], k, defaultValue); + m_table[index] = n; + m_tableCount++; + return &n->m_val; + } + + enum SetKind + { + None, + Overwrite + }; + //------------------------------------------------------------------------ // Set: Associate the specified value with the specified key. // @@ -279,12 +321,6 @@ class JitHashTable // If the key already exists and kind is Normal // this method will assert // - enum SetKind - { - None, - Overwrite - }; - bool Set(Key k, Value v, SetKind kind = None) { CheckGrowth(); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index ebcb21fff18bc..efd861348223d 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -8106,7 +8106,7 @@ void LinearScan::resolveRegisters() assert(currentRefPosition->isIntervalRef()); if (currentRefPosition->getInterval()->isInternal) { - treeNode->gtRsvdRegs |= currentRefPosition->registerAssignment; + compiler->codeGen->internalRegisters.Add(treeNode, currentRefPosition->registerAssignment); } else { @@ -8918,7 +8918,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) GenTree* switchTable = LIR::AsRange(block).LastNode(); assert(switchTable != nullptr && switchTable->OperGet() == GT_SWITCH_TABLE); - consumedRegs = switchTable->gtRsvdRegs; + consumedRegs = compiler->codeGen->internalRegisters.GetAll(switchTable); GenTree* op1 = switchTable->gtGetOp1(); GenTree* op2 = switchTable->gtGetOp2(); noway_assert(op1 != nullptr && op2 != nullptr); diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index c2b8b74406584..f88173e05e209 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -181,16 +181,24 @@ int LinearScan::BuildCall(GenTreeCall* call) } else if (call->IsR2ROrVirtualStubRelativeIndir()) { - // For R2R and VSD we have stub address in REG_R2R_INDIRECT_PARAM - // and will load call address into the temp register from this register. - regMaskTP candidates = RBM_NONE; if (call->IsFastTailCall()) { - candidates = allRegs(TYP_INT) & RBM_INT_CALLEE_TRASH; + // For R2R and VSD we have stub address in REG_R2R_INDIRECT_PARAM + // and will load call address into the temp register from this register. + regMaskTP candidates = allRegs(TYP_INT) & RBM_INT_CALLEE_TRASH; assert(candidates != RBM_NONE); + buildInternalIntRegisterDefForNode(call, candidates); + } + else + { + // For arm64 we can use lr for non-tailcalls so we skip the + // internal register as a TP optimization. We could do the same for + // arm32, but loading into lr cannot be encoded in 2 bytes, so + // another register is usually better. +#ifdef TARGET_ARM + buildInternalIntRegisterDefForNode(call); +#endif } - - buildInternalIntRegisterDefForNode(call, candidates); } #ifdef TARGET_ARM else diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 37fa4320cc691..22cba5900a7dd 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1730,10 +1730,6 @@ int LinearScan::ComputeAvailableSrcCount(GenTree* node) // void LinearScan::buildRefPositionsForNode(GenTree* tree, LsraLocation currentLoc) { - // The set of internal temporary registers used by this node are stored in the - // gtRsvdRegs register mask. Clear it out. - tree->gtRsvdRegs = RBM_NONE; - #ifdef DEBUG if (VERBOSE) { diff --git a/src/coreclr/jit/simdcodegenxarch.cpp b/src/coreclr/jit/simdcodegenxarch.cpp index d0fb1bf0aef4e..aff087062f58f 100644 --- a/src/coreclr/jit/simdcodegenxarch.cpp +++ b/src/coreclr/jit/simdcodegenxarch.cpp @@ -96,7 +96,7 @@ void CodeGen::genStoreIndTypeSimd12(GenTreeStoreInd* treeNode) } else { - regNumber tmpReg = treeNode->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(treeNode); // Extract upper 4 bytes from data emit->emitIns_R_R(INS_movhlps, EA_16BYTE, tmpReg, dataReg); @@ -295,7 +295,7 @@ void CodeGen::genEmitStoreLclTypeSimd12(GenTree* store, unsigned lclNum, unsigne } else { - regNumber tmpReg = store->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(store); // Extract upper 4 bytes from data emit->emitIns_R_R(INS_movhlps, EA_16BYTE, tmpReg, dataReg); @@ -391,7 +391,7 @@ void CodeGen::genPutArgStkSimd12(GenTreePutArgStk* treeNode) regNumber dataReg = genConsumeReg(data); // Need an additional Xmm register to extract upper 4 bytes from data. - regNumber tmpReg = treeNode->GetSingleTempReg(); + regNumber tmpReg = internalRegisters.GetSingle(treeNode); genStoreSimd12ToStack(dataReg, tmpReg); }