diff --git a/src/jit/CMakeLists.txt b/src/jit/CMakeLists.txt index f182d0beb47e..6372e3785275 100644 --- a/src/jit/CMakeLists.txt +++ b/src/jit/CMakeLists.txt @@ -63,6 +63,7 @@ set( JIT_SOURCES regset.cpp scopeinfo.cpp sharedfloat.cpp + sideeffects.cpp sm.cpp smdata.cpp smweights.cpp diff --git a/src/jit/codegenxarch.cpp b/src/jit/codegenxarch.cpp index cce0205f4686..a41c28695bd6 100644 --- a/src/jit/codegenxarch.cpp +++ b/src/jit/codegenxarch.cpp @@ -5133,13 +5133,13 @@ regNumber CodeGen::genConsumeReg(GenTree* tree) // Do liveness update for an address tree: one of GT_LEA, GT_LCL_VAR, or GT_CNS_INT (for call indirect). void CodeGen::genConsumeAddress(GenTree* addr) { - if (addr->OperGet() == GT_LEA) + if (!addr->isContained()) { - genConsumeAddrMode(addr->AsAddrMode()); + genConsumeReg(addr); } - else if (!addr->isContained()) + else if (addr->OperGet() == GT_LEA) { - genConsumeReg(addr); + genConsumeAddrMode(addr->AsAddrMode()); } } diff --git a/src/jit/gentree.cpp b/src/jit/gentree.cpp index bf6b3a179c25..67474e11ec90 100644 --- a/src/jit/gentree.cpp +++ b/src/jit/gentree.cpp @@ -1607,6 +1607,30 @@ regMaskTP GenTreeCall::GetOtherRegMask() const return resultMask; } +//------------------------------------------------------------------------- +// IsPure: +// Returns true if this call is pure. For now, this uses the same +// definition of "pure" that is that used by HelperCallProperties: a +// pure call does not read or write any aliased (e.g. heap) memory or +// have other global side effects (e.g. class constructors, finalizers), +// but is allowed to throw an exception. +// +// NOTE: this call currently only returns true if the call target is a +// helper method that is known to be pure. No other analysis is +// performed. +// +// Arguments: +// Copiler - the compiler context. +// +// Returns: +// True if the call is pure; false otherwise. +// +bool GenTreeCall::IsPure(Compiler* compiler) const +{ + return (gtCallType == CT_HELPER) && + compiler->s_helperCallProperties.IsPure(compiler->eeGetHelperNum(gtCallMethHnd)); +} + #ifndef LEGACY_BACKEND //------------------------------------------------------------------------- diff --git a/src/jit/gentree.h b/src/jit/gentree.h index 619c964f7126..4efeeae62088 100644 --- a/src/jit/gentree.h +++ b/src/jit/gentree.h @@ -1365,6 +1365,7 @@ struct GenTree { case GT_LOCKADD: case GT_XADD: + case GT_XCHG: case GT_CMPXCHG: case GT_BLK: case GT_OBJ: @@ -1404,7 +1405,7 @@ struct GenTree return (gtOper == GT_XADD || gtOper == GT_XCHG || gtOper == GT_LOCKADD || gtOper == GT_CMPXCHG); } - bool OperIsAtomicOp() + bool OperIsAtomicOp() const { return OperIsAtomicOp(gtOper); } @@ -3306,6 +3307,8 @@ struct GenTreeCall final : public GenTree return (gtCallMoreFlags & GTF_CALL_M_DOES_NOT_RETURN) != 0; } + bool IsPure(Compiler* compiler) const; + unsigned short gtCallMoreFlags; // in addition to gtFlags unsigned char gtCallType : 3; // value from the gtCallTypes enumeration diff --git a/src/jit/hashbv.cpp b/src/jit/hashbv.cpp index 32b286345bb3..fa06ec7b1eba 100644 --- a/src/jit/hashbv.cpp +++ b/src/jit/hashbv.cpp @@ -289,6 +289,19 @@ elemType hashBvNode::SubtractWithChange(hashBvNode* other) return result; } +bool hashBvNode::Intersects(hashBvNode* other) +{ + for (int i = 0; i < this->numElements(); i++) + { + if ((this->elements[i] & other->elements[i]) != 0) + { + return true; + } + } + + return false; +} + void hashBvNode::AndWith(hashBvNode* other) { for (int i = 0; i < this->numElements(); i++) @@ -1234,6 +1247,46 @@ class CompareAction } }; +class IntersectsAction +{ +public: + static inline void PreAction(hashBv* lhs, hashBv* rhs) + { + } + static inline void PostAction(hashBv* lhs, hashBv* rhs) + { + } + static inline bool DefaultResult() + { + return false; + } + + static inline void LeftGap(hashBv* lhs, hashBvNode**& l, hashBvNode*& r, bool& result, bool& terminate) + { + // in rhs, not lhs + // so skip rhs + r = r->next; + } + static inline void RightGap(hashBv* lhs, hashBvNode**& l, hashBvNode*& r, bool& result, bool& terminate) + { + // in lhs, not rhs + // so skip lhs + l = &((*l)->next); + } + static inline void BothPresent(hashBv* lhs, hashBvNode**& l, hashBvNode*& r, bool& result, bool& terminate) + { + if ((*l)->Intersects(r)) + { + terminate = true; + result = true; + } + } + static inline void LeftEmpty(hashBv* lhs, hashBvNode**& l, hashBvNode*& r, bool& result, bool& terminate) + { + r = r->next; + } +}; + template bool hashBv::MultiTraverseLHSBigger(hashBv* other) { @@ -1507,6 +1560,11 @@ bool hashBv::MultiTraverse(hashBv* other) } } +bool hashBv::Intersects(hashBv* other) +{ + return MultiTraverse(other); +} + bool hashBv::AndWithChange(hashBv* other) { return MultiTraverse(other); diff --git a/src/jit/hashbv.h b/src/jit/hashbv.h index 55d3b27bddb1..cadb182cc663 100644 --- a/src/jit/hashbv.h +++ b/src/jit/hashbv.h @@ -157,6 +157,8 @@ class hashBvNode elemType XorWithChange(hashBvNode* other); elemType SubtractWithChange(hashBvNode* other); + bool Intersects(hashBvNode* other); + #ifdef DEBUG void dump(); #endif // DEBUG @@ -253,6 +255,8 @@ class hashBv bool XorWithChange(hashBv* other); bool SubtractWithChange(hashBv* other); + bool Intersects(hashBv* other); + template bool MultiTraverseLHSBigger(hashBv* other); template diff --git a/src/jit/jit.settings.targets b/src/jit/jit.settings.targets index 27c7680ee505..9dbc22584341 100644 --- a/src/jit/jit.settings.targets +++ b/src/jit/jit.settings.targets @@ -86,6 +86,7 @@ + diff --git a/src/jit/lower.cpp b/src/jit/lower.cpp index c09541124506..09eb9146aca8 100644 --- a/src/jit/lower.cpp +++ b/src/jit/lower.cpp @@ -78,34 +78,20 @@ bool Lowering::CheckImmedAndMakeContained(GenTree* parentNode, GenTree* childNod // and returns 'true' iff memory operand childNode can be contained in parentNode. // // Arguments: -// parentNode - a non-leaf binary node -// childNode - a memory op that is a child op of 'parentNode' +// parentNode - any non-leaf node +// childNode - some node that is an input to `parentNode` // // Return value: // true if it is safe to make childNode a contained memory operand. // bool Lowering::IsSafeToContainMem(GenTree* parentNode, GenTree* childNode) { - assert(parentNode->OperIsBinary()); - assert(childNode->isMemoryOp()); + m_scratchSideEffects.Clear(); + m_scratchSideEffects.AddNode(comp, childNode); - unsigned int childFlags = (childNode->gtFlags & GTF_ALL_EFFECT); - - GenTree* node; - for (node = childNode; node != parentNode; node = node->gtNext) + for (GenTree* node = childNode->gtNext; node != parentNode; node = node->gtNext) { - assert(node != nullptr); - - if ((childFlags != 0) && node->IsCall()) - { - bool isPureHelper = (node->gtCall.gtCallType == CT_HELPER) && - comp->s_helperCallProperties.IsPure(comp->eeGetHelperNum(node->gtCall.gtCallMethHnd)); - if (!isPureHelper && ((node->gtFlags & childFlags & GTF_ALL_EFFECT) != 0)) - { - return false; - } - } - else if (node->OperIsStore() && comp->fgNodesMayInterfere(node, childNode)) + if (m_scratchSideEffects.InterferesWith(comp, node, false)) { return false; } @@ -2978,17 +2964,57 @@ void Lowering::AddrModeCleanupHelper(GenTreeAddrMode* addrMode, GenTree* node) BlockRange().Remove(node); } -// given two nodes which will be used in an addressing mode (base, index) -// walk backwards from the use to those nodes to determine if they are -// potentially modified in that range +//------------------------------------------------------------------------ +// Lowering::AreSourcesPossibleModifiedLocals: +// Given two nodes which will be used in an addressing mode (base, +// index), check to see if they are lclVar reads, and if so, walk +// backwards from the use until both reads have been visited to +// determine if they are potentially modified in that range. +// +// Arguments: +// addr - the node that uses the base and index nodes +// base - the base node +// index - the index node +// +// Returns: true if either the base or index may be modified between the +// node and addr. // -// returns: true if the sources given may be modified before they are used -bool Lowering::AreSourcesPossiblyModified(GenTree* addr, GenTree* base, GenTree* index) +bool Lowering::AreSourcesPossiblyModifiedLocals(GenTree* addr, GenTree* base, GenTree* index) { assert(addr != nullptr); - for (GenTree* cursor = addr; cursor != nullptr; cursor = cursor->gtPrev) + unsigned markCount = 0; + + SideEffectSet baseSideEffects; + if (base != nullptr) { + if (base->OperIsLocalRead()) + { + baseSideEffects.AddNode(comp, base); + } + else + { + base = nullptr; + } + } + + SideEffectSet indexSideEffects; + if (index != nullptr) + { + if (index->OperIsLocalRead()) + { + indexSideEffects.AddNode(comp, index); + } + else + { + index = nullptr; + } + } + + for (GenTree* cursor = addr;; cursor = cursor->gtPrev) + { + assert(cursor != nullptr); + if (cursor == base) { base = nullptr; @@ -2999,17 +3025,19 @@ bool Lowering::AreSourcesPossiblyModified(GenTree* addr, GenTree* base, GenTree* index = nullptr; } - if (base == nullptr && index == nullptr) + if ((base == nullptr) && (index == nullptr)) { return false; } - if (base != nullptr && comp->fgNodesMayInterfere(base, cursor)) + m_scratchSideEffects.Clear(); + m_scratchSideEffects.AddNode(comp, cursor); + if ((base != nullptr) && m_scratchSideEffects.InterferesWith(baseSideEffects, false)) { return true; } - if (index != nullptr && comp->fgNodesMayInterfere(index, cursor)) + if ((index != nullptr) && m_scratchSideEffects.InterferesWith(indexSideEffects, false)) { return true; } @@ -3092,7 +3120,7 @@ GenTree* Lowering::TryCreateAddrMode(LIR::Use&& use, bool isIndir) } // make sure there are not any side effects between def of leaves and use - if (!doAddrMode || AreSourcesPossiblyModified(addr, base, index)) + if (!doAddrMode || AreSourcesPossiblyModifiedLocals(addr, base, index)) { JITDUMP(" No addressing mode\n"); return addr; @@ -3122,7 +3150,8 @@ GenTree* Lowering::TryCreateAddrMode(LIR::Use&& use, bool isIndir) GenTreeAddrMode* addrMode = new (comp, GT_LEA) GenTreeAddrMode(addrModeType, base, index, scale, offset); addrMode->gtRsvdRegs = addr->gtRsvdRegs; - addrMode->gtFlags |= (addr->gtFlags & (GTF_ALL_EFFECT | GTF_IND_FLAGS)); + addrMode->gtFlags |= (addr->gtFlags & GTF_IND_FLAGS); + addrMode->gtFlags &= ~GTF_ALL_EFFECT; // LEAs are side-effect-free. JITDUMP("New addressing mode node:\n"); DISPNODE(addrMode); diff --git a/src/jit/lower.h b/src/jit/lower.h index 1450f03d4a63..620636d8bd6e 100644 --- a/src/jit/lower.h +++ b/src/jit/lower.h @@ -17,6 +17,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #include "compiler.h" #include "phase.h" #include "lsra.h" +#include "sideeffects.h" class Lowering : public Phase { @@ -228,6 +229,7 @@ class Lowering : public Phase void LowerCmp(GenTreePtr tree); #if !CPU_LOAD_STORE_ARCH + bool IsRMWIndirCandidate(GenTree* operand, GenTree* storeInd); bool IsBinOpInRMWStoreInd(GenTreePtr tree); bool IsRMWMemOpRootedAtStoreInd(GenTreePtr storeIndTree, GenTreePtr* indirCandidate, GenTreePtr* indirOpSource); bool SetStoreIndOpCountsIfRMWMemOp(GenTreePtr storeInd); @@ -247,7 +249,7 @@ class Lowering : public Phase private: static bool NodesAreEquivalentLeaves(GenTreePtr candidate, GenTreePtr storeInd); - bool AreSourcesPossiblyModified(GenTree* addr, GenTree* base, GenTree* index); + bool AreSourcesPossiblyModifiedLocals(GenTree* addr, GenTree* base, GenTree* index); // return true if 'childNode' is an immediate that can be contained // by the 'parentNode' (i.e. folded into an instruction) @@ -269,9 +271,10 @@ class Lowering : public Phase return LIR::AsRange(m_block); } - LinearScan* m_lsra; - unsigned vtableCallTemp; // local variable we use as a temp for vtable calls - BasicBlock* m_block; + LinearScan* m_lsra; + unsigned vtableCallTemp; // local variable we use as a temp for vtable calls + SideEffectSet m_scratchSideEffects; // SideEffectSet used for IsSafeToContainMem and isRMWIndirCandidate + BasicBlock* m_block; }; #endif // _LOWER_H_ diff --git a/src/jit/lowerarm.cpp b/src/jit/lowerarm.cpp index 4c9648598c10..67cea2ff4e19 100644 --- a/src/jit/lowerarm.cpp +++ b/src/jit/lowerarm.cpp @@ -28,6 +28,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #ifdef _TARGET_ARM_ #include "jit.h" +#include "sideeffects.h" #include "lower.h" #include "lsra.h" diff --git a/src/jit/lowerarm64.cpp b/src/jit/lowerarm64.cpp index 709639d93187..1720c62acb4e 100644 --- a/src/jit/lowerarm64.cpp +++ b/src/jit/lowerarm64.cpp @@ -26,6 +26,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #ifdef _TARGET_ARM64_ #include "jit.h" +#include "sideeffects.h" #include "lower.h" // there is not much lowering to do with storing a local but @@ -1735,7 +1736,7 @@ void Lowering::SetIndirAddrOpCounts(GenTreePtr indirTree) bool rev; bool modifiedSources = false; - if (addr->OperGet() == GT_LEA) + if ((addr->OperGet() == GT_LEA) && IsSafeToContainMem(indirTree, addr)) { GenTreeAddrMode* lea = addr->AsAddrMode(); base = lea->Base(); @@ -1748,7 +1749,7 @@ void Lowering::SetIndirAddrOpCounts(GenTreePtr indirTree) info->srcCount--; } else if (comp->codeGen->genCreateAddrMode(addr, -1, true, 0, &rev, &base, &index, &mul, &cns, true /*nogen*/) && - !(modifiedSources = AreSourcesPossiblyModified(indirTree, base, index))) + !(modifiedSources = AreSourcesPossiblyModifiedLocals(indirTree, base, index))) { // An addressing mode will be constructed that may cause some // nodes to not need a register, and cause others' lifetimes to be extended diff --git a/src/jit/lowerxarch.cpp b/src/jit/lowerxarch.cpp index 3bbc75baf5cf..6f98eb66619c 100644 --- a/src/jit/lowerxarch.cpp +++ b/src/jit/lowerxarch.cpp @@ -26,6 +26,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #ifdef _TARGET_XARCH_ #include "jit.h" +#include "sideeffects.h" #include "lower.h" // xarch supports both ROL and ROR instructions so no lowering is required. @@ -2752,7 +2753,7 @@ void Lowering::SetIndirAddrOpCounts(GenTreePtr indirTree) // On x86, direct VSD is done via a relative branch, and in fact it MUST be contained. MakeSrcContained(indirTree, addr); } - else if (addr->OperGet() == GT_LEA) + else if ((addr->OperGet() == GT_LEA) && IsSafeToContainMem(indirTree, addr)) { GenTreeAddrMode* lea = addr->AsAddrMode(); base = lea->Base(); @@ -2764,7 +2765,7 @@ void Lowering::SetIndirAddrOpCounts(GenTreePtr indirTree) info->srcCount--; } else if (comp->codeGen->genCreateAddrMode(addr, -1, true, 0, &rev, &base, &index, &mul, &cns, true /*nogen*/) && - !(modifiedSources = AreSourcesPossiblyModified(indirTree, base, index))) + !(modifiedSources = AreSourcesPossiblyModifiedLocals(indirTree, base, index))) { // An addressing mode will be constructed that may cause some // nodes to not need a register, and cause others' lifetimes to be extended @@ -3373,6 +3374,86 @@ void Lowering::LowerCast(GenTree* tree) } } +//---------------------------------------------------------------------------------------------- +// Lowering::IsRMWIndirCandidate: +// Returns true if the given operand is a candidate indirection for a read-modify-write +// operator. +// +// Arguments: +// operand - The operand to consider. +// storeInd - The indirect store that roots the possible RMW operator. +// +bool Lowering::IsRMWIndirCandidate(GenTree* operand, GenTree* storeInd) +{ + // If the operand isn't an indirection, it's trivially not a candidate. + if (operand->OperGet() != GT_IND) + { + return false; + } + + // If the indirection's source address isn't equivalent to the destination address of the storeIndir, then the + // indirection is not a candidate. + GenTree* srcAddr = operand->gtGetOp1(); + GenTree* dstAddr = storeInd->gtGetOp1(); + if ((srcAddr->OperGet() != dstAddr->OperGet()) || !IndirsAreEquivalent(operand, storeInd)) + { + return false; + } + + // If it is not safe to contain the entire tree rooted at the indirection, then the indirection is not a + // candidate. Crawl the IR from the node immediately preceding the storeIndir until the last node in the + // indirection's tree is visited and check the side effects at each point. + + m_scratchSideEffects.Clear(); + + assert((operand->gtLIRFlags & LIR::Flags::Mark) == 0); + operand->gtLIRFlags |= LIR::Flags::Mark; + + unsigned markCount = 1; + GenTree* node; + for (node = storeInd->gtPrev; markCount > 0; node = node->gtPrev) + { + assert(node != nullptr); + + if ((node->gtLIRFlags & LIR::Flags::Mark) == 0) + { + m_scratchSideEffects.AddNode(comp, node); + } + else + { + node->gtLIRFlags &= ~LIR::Flags::Mark; + markCount--; + + if (m_scratchSideEffects.InterferesWith(comp, node, false)) + { + // The indirection's tree contains some node that can't be moved to the storeInder. The indirection is + // not a candidate. Clear any leftover mark bits and return. + for (; markCount > 0; node = node->gtPrev) + { + if ((node->gtLIRFlags & LIR::Flags::Mark) != 0) + { + node->gtLIRFlags &= ~LIR::Flags::Mark; + markCount--; + } + } + return false; + } + + for (GenTree* nodeOperand : node->Operands()) + { + assert((nodeOperand->gtLIRFlags & LIR::Flags::Mark) == 0); + nodeOperand->gtLIRFlags |= LIR::Flags::Mark; + markCount++; + } + } + } + + // At this point we've verified that the operand is an indirection, its address is equivalent to the storeIndir's + // destination address, and that it and the transitive closure of its operand can be safely contained by the + // storeIndir. This indirection is therefore a candidate for an RMW op. + return true; +} + //---------------------------------------------------------------------------------------------- // Returns true if this tree is bin-op of a GT_STOREIND of the following form // storeInd(subTreeA, binOp(gtInd(subTreeA), subtreeB)) or @@ -3527,6 +3608,28 @@ bool Lowering::IsRMWMemOpRootedAtStoreInd(GenTreePtr tree, GenTreePtr* outIndirC return false; } + // At this point we can match one of two patterns: + // + // t_ind = indir t_addr_0 + // ... + // t_value = binop t_ind, t_other + // ... + // storeIndir t_addr_1, t_value + // + // or + // + // t_ind = indir t_addr_0 + // ... + // t_value = unop t_ind + // ... + // storeIndir t_addr_1, t_value + // + // In all cases, we will eventually make the binop that produces t_value and the entire dataflow tree rooted at + // t_ind contained by t_value. + + GenTree* indirCandidate = nullptr; + GenTree* indirOpSource = nullptr; + RMWStatus status = STOREIND_RMW_STATUS_UNKNOWN; if (GenTree::OperIsBinary(oper)) { // Return if binary op is not one of the supported operations for RMW of memory. @@ -3546,29 +3649,25 @@ bool Lowering::IsRMWMemOpRootedAtStoreInd(GenTreePtr tree, GenTreePtr* outIndirC return false; } - GenTreePtr rhsLeft = indirSrc->gtGetOp1(); - GenTreePtr rhsRight = indirSrc->gtGetOp2(); - - // The most common case is rhsRight is GT_IND - if (GenTree::OperIsCommutative(oper) && rhsRight->OperGet() == GT_IND && - rhsRight->gtGetOp1()->OperGet() == indirDst->OperGet() && IndirsAreEquivalent(rhsRight, storeInd)) + // In the common case, the second operand to the binop will be the indir candidate. + GenTreeOp* binOp = indirSrc->AsOp(); + if (GenTree::OperIsCommutative(oper) && IsRMWIndirCandidate(binOp->gtOp2, storeInd)) { - *outIndirCandidate = rhsRight; - *outIndirOpSource = rhsLeft; - storeInd->SetRMWStatus(STOREIND_RMW_DST_IS_OP2); - return true; + indirCandidate = binOp->gtOp2; + indirOpSource = binOp->gtOp1; + status = STOREIND_RMW_DST_IS_OP2; } - else if (rhsLeft->OperGet() == GT_IND && rhsLeft->gtGetOp1()->OperGet() == indirDst->OperGet() && - IsSafeToContainMem(indirSrc, rhsLeft) && IndirsAreEquivalent(rhsLeft, storeInd)) + else if (IsRMWIndirCandidate(binOp->gtOp1, storeInd)) { - *outIndirCandidate = rhsLeft; - *outIndirOpSource = rhsRight; - storeInd->SetRMWStatus(STOREIND_RMW_DST_IS_OP1); - return true; + indirCandidate = binOp->gtOp1; + indirOpSource = binOp->gtOp2; + status = STOREIND_RMW_DST_IS_OP1; + } + else + { + storeInd->SetRMWStatus(STOREIND_RMW_UNSUPPORTED_ADDR); + return false; } - - storeInd->SetRMWStatus(STOREIND_RMW_UNSUPPORTED_ADDR); - return false; } else if (GenTree::OperIsUnary(oper)) { @@ -3585,22 +3684,42 @@ bool Lowering::IsRMWMemOpRootedAtStoreInd(GenTreePtr tree, GenTreePtr* outIndirC return false; } - GenTreePtr indirCandidate = indirSrc->gtGetOp1(); - if (indirCandidate->gtGetOp1()->OperGet() == indirDst->OperGet() && - IndirsAreEquivalent(indirCandidate, storeInd)) + GenTreeUnOp* unOp = indirSrc->AsUnOp(); + if (IsRMWIndirCandidate(unOp->gtOp1, storeInd)) { // src and dest are the same in case of unary ops - *outIndirCandidate = indirCandidate; - *outIndirOpSource = indirCandidate; - storeInd->SetRMWStatus(STOREIND_RMW_DST_IS_OP1); - return true; + indirCandidate = unOp->gtOp1; + indirOpSource = unOp->gtOp1; + status = STOREIND_RMW_DST_IS_OP1; + } + else + { + storeInd->SetRMWStatus(STOREIND_RMW_UNSUPPORTED_ADDR); + return false; } } + else + { + storeInd->SetRMWStatus(STOREIND_RMW_UNSUPPORTED_OPER); + return false; + } + + // By this point we've verified that we have a supported operand with a supported address. Now we need to ensure + // that we're able to move the destination address for the source indirection forwards. + if (!IsSafeToContainMem(storeInd, indirDst)) + { + storeInd->SetRMWStatus(STOREIND_RMW_UNSUPPORTED_ADDR); + return false; + } - assert(*outIndirCandidate == nullptr); - assert(*outIndirOpSource == nullptr); - storeInd->SetRMWStatus(STOREIND_RMW_UNSUPPORTED_OPER); - return false; + assert(indirCandidate != nullptr); + assert(indirOpSource != nullptr); + assert(status != STOREIND_RMW_STATUS_UNKNOWN); + + *outIndirCandidate = indirCandidate; + *outIndirOpSource = indirOpSource; + storeInd->SetRMWStatus(status); + return true; } //-------------------------------------------------------------------------------------------- diff --git a/src/jit/sideeffects.cpp b/src/jit/sideeffects.cpp new file mode 100644 index 000000000000..dbfa27cfae5f --- /dev/null +++ b/src/jit/sideeffects.cpp @@ -0,0 +1,549 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#include "jitpch.h" +#ifdef _MSC_VER +#pragma hdrstop +#endif + +#include "sideeffects.h" + +LclVarSet::LclVarSet() : m_bitVector(nullptr), m_hasAnyLcl(false), m_hasBitVector(false) +{ +} + +//------------------------------------------------------------------------ +// LclVarSet::Add: +// Adds the given lclNum to the LclVarSet. +// +// Arguments: +// compiler - The compiler context +// lclNum - The lclNum to add. +// +void LclVarSet::Add(Compiler* compiler, unsigned lclNum) +{ + if (!m_hasAnyLcl) + { + m_lclNum = lclNum; + m_hasAnyLcl = true; + } + else + { + if (!m_hasBitVector) + { + unsigned singleLclNum = m_lclNum; + m_bitVector = hashBv::Create(compiler); + m_bitVector->setBit(singleLclNum); + m_hasBitVector = true; + } + + m_bitVector->setBit(lclNum); + } +} + +//------------------------------------------------------------------------ +// LclVarSet::Intersects: +// Returns true if this LclVarSet intersects with the given LclVarSet. +// +// Arguments: +// other - The other lclVarSet. +// +bool LclVarSet::Intersects(const LclVarSet& other) const +{ + // If neither set has ever contained anything, the sets do not intersect. + if (!m_hasAnyLcl || !other.m_hasAnyLcl) + { + return false; + } + + // If this set is not represented by a bit vector, see if the single lclNum is contained in the other set. + if (!m_hasBitVector) + { + if (!other.m_hasBitVector) + { + return m_lclNum == other.m_lclNum; + } + + return other.m_bitVector->testBit(m_lclNum); + } + + // If this set is represented by a bit vector but the other set is not, see if the single lclNum in the other + // set is contained in this set. + if (!other.m_hasBitVector) + { + return m_bitVector->testBit(other.m_lclNum); + } + + // Both sets are represented by bit vectors. Check to see if they intersect. + return m_bitVector->Intersects(other.m_bitVector); +} + +//------------------------------------------------------------------------ +// LclVarSet::Contains: +// Returns true if this LclVarSet contains the given lclNum. +// +// Arguments: +// lclNum - The lclNum in question. +// +bool LclVarSet::Contains(unsigned lclNum) const +{ + // If this set has never contained anything, it does not contain the lclNum. + if (!m_hasAnyLcl) + { + return false; + } + + // If this set is not represented by a bit vector, see if its single lclNum is the same as the given lclNum. + if (!m_hasBitVector) + { + return m_lclNum == lclNum; + } + + // This set is represented by a bit vector. See if the bit vector contains the given lclNum. + return m_bitVector->testBit(lclNum); +} + +//------------------------------------------------------------------------ +// LclVarSet::Clear: +// Clears the contents of this LclVarSet. +// +void LclVarSet::Clear() +{ + if (m_hasBitVector) + { + assert(m_hasAnyLcl); + m_bitVector->ZeroAll(); + } + else if (m_hasAnyLcl) + { + m_hasAnyLcl = false; + } +} + +AliasSet::AliasSet() + : m_lclVarReads(), m_lclVarWrites(), m_readsAddressableLocation(false), m_writesAddressableLocation(false) +{ +} + +//------------------------------------------------------------------------ +// AliasSet::NodeInfo::NodeInfo: +// Computes the alias info for a given node. Note that this does not +// include the set of lclVar accesses for a node unless the node is +// itself a lclVar access (e.g. a GT_LCL_VAR, GT_STORE_LCL_VAR, etc.). +// +// Arguments: +// compiler - The compiler context. +// node - The node in question. +// +AliasSet::NodeInfo::NodeInfo(Compiler* compiler, GenTree* node) + : m_compiler(compiler), m_node(node), m_flags(0), m_lclNum(0) +{ + if (node->IsCall()) + { + // Calls are treated as reads and writes of addressable locations unless they are known to be pure. + if (node->AsCall()->IsPure(compiler)) + { + m_flags = ALIAS_NONE; + return; + } + + m_flags = ALIAS_READS_ADDRESSABLE_LOCATION | ALIAS_WRITES_ADDRESSABLE_LOCATION; + return; + } + else if (node->OperIsAtomicOp()) + { + // Atomic operations both read and write addressable locations. + m_flags = ALIAS_READS_ADDRESSABLE_LOCATION | ALIAS_WRITES_ADDRESSABLE_LOCATION; + return; + } + + // Is the operation a write? If so, set `node` to the location that is being written to. + bool isWrite = false; + if (node->OperIsAssignment()) + { + isWrite = true; + node = node->gtGetOp1(); + } + else if (node->OperIsStore() || node->OperIsAtomicOp()) + { + isWrite = true; + } + + // `node` is the location being accessed. Determine whether or not it is a memory or local variable access, and if + // it is the latter, get the number of the lclVar. + bool isMemoryAccess = false; + bool isLclVarAccess = false; + unsigned lclNum = 0; + if (node->OperIsIndir()) + { + // If the indirection targets a lclVar, we can be more precise with regards to aliasing by treating the + // indirection as a lclVar access. + GenTree* address = node->AsIndir()->Addr(); + if (address->OperIsLocalAddr()) + { + isLclVarAccess = true; + lclNum = address->AsLclVarCommon()->GetLclNum(); + } + else + { + isMemoryAccess = true; + } + } + else if (node->OperIsImplicitIndir()) + { + isMemoryAccess = true; + } + else if (node->OperIsLocal()) + { + isLclVarAccess = true; + lclNum = node->AsLclVarCommon()->GetLclNum(); + } + else + { + // This is neither a memory nor a local var access. + m_flags = ALIAS_NONE; + return; + } + + assert(isMemoryAccess || isLclVarAccess); + + // Now that we've determined whether or not this access is a read or a write and whether the accessed location is + // memory or a lclVar, determine whther or not the location is addressable and udpate the alias set. + const bool isAddressableLocation = isMemoryAccess || compiler->lvaTable[lclNum].lvAddrExposed; + + if (!isWrite) + { + if (isAddressableLocation) + { + m_flags |= ALIAS_READS_ADDRESSABLE_LOCATION; + } + + if (isLclVarAccess) + { + m_flags |= ALIAS_READS_LCL_VAR; + m_lclNum = lclNum; + } + } + else + { + if (isAddressableLocation) + { + m_flags |= ALIAS_WRITES_ADDRESSABLE_LOCATION; + } + + if (isLclVarAccess) + { + m_flags |= ALIAS_WRITES_LCL_VAR; + m_lclNum = lclNum; + } + } +} + +//------------------------------------------------------------------------ +// AliasSet::AddNode: +// Adds the given node's accesses to this AliasSet. +// +// Arguments: +// compiler - The compiler context. +// node - The node to add to the set. +// +void AliasSet::AddNode(Compiler* compiler, GenTree* node) +{ + // First, add all lclVar uses associated with the node to the set. This is necessary because the lclVar reads occur + // at the position of the user, not at the position of the GenTreeLclVar node. + for (GenTree* operand : node->Operands()) + { + if (operand->OperIsLocalRead()) + { + const unsigned lclNum = operand->AsLclVarCommon()->GetLclNum(); + if (compiler->lvaTable[lclNum].lvAddrExposed) + { + m_readsAddressableLocation = true; + } + + m_lclVarReads.Add(compiler, lclNum); + } + } + + NodeInfo nodeInfo(compiler, node); + if (nodeInfo.ReadsAddressableLocation()) + { + m_readsAddressableLocation = true; + } + if (nodeInfo.WritesAddressableLocation()) + { + m_writesAddressableLocation = true; + } + if (nodeInfo.IsLclVarRead()) + { + m_lclVarReads.Add(compiler, nodeInfo.LclNum()); + } + if (nodeInfo.IsLclVarWrite()) + { + m_lclVarWrites.Add(compiler, nodeInfo.LclNum()); + } +} + +//------------------------------------------------------------------------ +// AliasSet::InterferesWith: +// Returns true if the reads and writes in this alias set interfere +// with the given alias set. +// +// Two alias sets interfere under any of the following conditions: +// - Both sets write to any addressable location (e.g. the heap, +// address-exposed locals) +// - One set reads any addressable location and the other set writes +// any addressable location +// - Both sets write to the same lclVar +// - One set writes to a lclVar that is read by the other set +// +// Arguments: +// other - The other alias set. +// +bool AliasSet::InterferesWith(const AliasSet& other) const +{ + // If both sets write any addressable location, the sets interfere. + if (m_writesAddressableLocation && other.m_writesAddressableLocation) + { + return true; + } + + // If one set writes any addressable location and the other reads any addressable location, the sets interfere. + if ((m_readsAddressableLocation && other.m_writesAddressableLocation) || + (m_writesAddressableLocation && other.m_readsAddressableLocation)) + { + return true; + } + + // If the set of lclVars written by this alias set intersects with the set of lclVars accessed by the other alias + // set, the alias sets interfere. + if (m_lclVarWrites.Intersects(other.m_lclVarReads) || m_lclVarWrites.Intersects(other.m_lclVarWrites)) + { + return true; + } + + // If the set of lclVars read by this alias set intersects with the set of lclVars written by the other alias set, + // the alias sets interfere. Otherwise, the alias sets do not interfere. + return m_lclVarReads.Intersects(other.m_lclVarWrites); +} + +//------------------------------------------------------------------------ +// AliasSet::InterferesWith: +// Returns true if the reads and writes in this alias set interfere +// with those for the given node. +// +// An alias set interferes with a given node iff it interferes with the +// alias set for that node. +// +// Arguments: +// other - The info for the node in question. +// +bool AliasSet::InterferesWith(const NodeInfo& other) const +{ + // First check whether or not this set interferes with the lclVar uses associated with the given node. + if (m_writesAddressableLocation || !m_lclVarWrites.IsEmpty()) + { + Compiler* compiler = other.TheCompiler(); + for (GenTree* operand : other.Node()->Operands()) + { + if (operand->OperIsLocalRead()) + { + // If this set writes any addressable location and the node uses an address-exposed lclVar, + // the set interferes with the node. + const unsigned lclNum = operand->AsLclVarCommon()->GetLclNum(); + if (compiler->lvaTable[lclNum].lvAddrExposed && m_writesAddressableLocation) + { + return true; + } + + // If this set writes to a lclVar used by the node, the set interferes with the node. + if (m_lclVarWrites.Contains(lclNum)) + { + return true; + } + } + } + } + + // If the node and the set both write to any addressable location, they interfere. + if (m_writesAddressableLocation && other.WritesAddressableLocation()) + { + return true; + } + + // If the node or the set writes any addressable location and the other reads any addressable location, + // they interfere. + if ((m_readsAddressableLocation && other.WritesAddressableLocation()) || + (m_writesAddressableLocation && other.ReadsAddressableLocation())) + { + return true; + } + + // If the set writes a local var accessed by the node, they interfere. + if ((other.IsLclVarRead() || other.IsLclVarWrite()) && m_lclVarWrites.Contains(other.LclNum())) + { + return true; + } + + // If the set reads a local var written by the node, they interfere. + return other.IsLclVarWrite() && m_lclVarReads.Contains(other.LclNum()); +} + +//------------------------------------------------------------------------ +// AliasSet::Clear: +// Clears the current alias set. +// +void AliasSet::Clear() +{ + m_readsAddressableLocation = false; + m_writesAddressableLocation = false; + + m_lclVarReads.Clear(); + m_lclVarWrites.Clear(); +} + +SideEffectSet::SideEffectSet() : m_sideEffectFlags(0), m_aliasSet() +{ +} + +//------------------------------------------------------------------------ +// SideEffectSet::SideEffectSet: +// Constructs a side effect set initialized using the given node. +// Equivalent to the following; +// +// SideEffectSet sideEffectSet; +// sideEffectSet.AddNode(compiler, node); +// +// Arguments: +// compiler - The compiler context. +// node - The node to use for initialization. +// +SideEffectSet::SideEffectSet(Compiler* compiler, GenTree* node) : m_sideEffectFlags(0), m_aliasSet() +{ + AddNode(compiler, node); +} + +//------------------------------------------------------------------------ +// SideEffectSet::AddNode: +// Adds the given node's accesses to this SideEffectSet. +// +// Arguments: +// compiler - The compiler context. +// node - The node to add to the set. +// +void SideEffectSet::AddNode(Compiler* compiler, GenTree* node) +{ + m_sideEffectFlags |= (node->gtFlags & GTF_ALL_EFFECT); + m_aliasSet.AddNode(compiler, node); +} + +//------------------------------------------------------------------------ +// SideEffectSet::InterferesWith: +// Returns true if the side effects in this set interfere with the +// given side effect flags and alias information. +// +// Two side effect sets interfere under any of the following +// conditions: +// - If the analysis is strict, and: +// - Either set contains a compiler barrier, or +// - Both sets produce an exception +// - Whether or not the analysis is strict: +// - One set produces an exception and the other set contains a +// write +// - One set's reads and writes interfere with the other set's +// reads and writes +// +// Arguments: +// otherSideEffectFlags - The side effect flags for the other side +// effect set. +// otherAliasInfo - The alias information for the other side effect +// set. +// strict - True if the analysis should be strict as described above. +// +template +bool SideEffectSet::InterferesWith(unsigned otherSideEffectFlags, + const TOtherAliasInfo& otherAliasInfo, + bool strict) const +{ + const bool thisProducesException = (m_sideEffectFlags & GTF_EXCEPT) != 0; + const bool otherProducesException = (otherSideEffectFlags & GTF_EXCEPT) != 0; + + if (strict) + { + // If either set contains a compiler barrier, the sets interfere. + if (((m_sideEffectFlags | otherSideEffectFlags) & GTF_ORDER_SIDEEFF) != 0) + { + return true; + } + + // If both sets produce an exception, the sets interfere. + if (thisProducesException && otherProducesException) + { + return true; + } + } + + // If one set produces an exception and the other set writes to any location, the sets interfere. + if ((thisProducesException && otherAliasInfo.WritesAnyLocation()) || + (otherProducesException && m_aliasSet.WritesAnyLocation())) + { + return true; + } + + // At this point, the only interference between the sets will arise from their alias sets. + return m_aliasSet.InterferesWith(otherAliasInfo); +} + +//------------------------------------------------------------------------ +// SideEffectSet::InterferesWith: +// Returns true if the side effects in this set interfere with the side +// effects in the given side effect set. +// +// Two side effect sets interfere under any of the following +// conditions: +// - If the analysis is strict, and: +// - Either set contains a compiler barrier, or +// - Both sets produce an exception +// - Whether or not the analysis is strict: +// - One set produces an exception and the other set contains a +// write +// - One set's reads and writes interfere with the other set's +// reads and writes +// +// Arguments: +// other - The other side effect set. +// strict - True if the analysis should be strict as described above. +// +bool SideEffectSet::InterferesWith(const SideEffectSet& other, bool strict) const +{ + return InterferesWith(other.m_sideEffectFlags, other.m_aliasSet, strict); +} + +//------------------------------------------------------------------------ +// SideEffectSet::InterferesWith: +// Returns true if the side effects in this set interfere with the side +// effects for the given node. +// +// A side effect set interferes with a given node iff it interferes +// with the side effect set of the node. +// +// Arguments: +// compiler - The compiler context. +// node - The node in question. +// strict - True if the analysis should be strict as described above. +// +bool SideEffectSet::InterferesWith(Compiler* compiler, GenTree* node, bool strict) const +{ + return InterferesWith((node->gtFlags & GTF_ALL_EFFECT), AliasSet::NodeInfo(compiler, node), strict); +} + +//------------------------------------------------------------------------ +// SideEffectSet::Clear: +// Clears the current side effect set. +// +void SideEffectSet::Clear() +{ + m_sideEffectFlags = 0; + m_aliasSet.Clear(); +} diff --git a/src/jit/sideeffects.h b/src/jit/sideeffects.h new file mode 100644 index 000000000000..33fac16f057a --- /dev/null +++ b/src/jit/sideeffects.h @@ -0,0 +1,158 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#ifndef _SIDEEFFECTS_H_ +#define _SIDEEFFECTS_H_ + +//------------------------------------------------------------------------ +// LclVarSet: +// Represents a set of lclVars. Optimized for the case that the set +// never holds more than a single element. This type is used internally +// by `AliasSet` to track the sets of lclVars that are read and +// written for a given alias set. +// +class LclVarSet final +{ + union { + hashBv* m_bitVector; + unsigned m_lclNum; + }; + + bool m_hasAnyLcl; + bool m_hasBitVector; + +public: + LclVarSet(); + + inline bool IsEmpty() const + { + return !m_hasAnyLcl || !m_hasBitVector || !m_bitVector->anySet(); + } + + void Add(Compiler* compiler, unsigned lclNum); + bool Intersects(const LclVarSet& other) const; + bool Contains(unsigned lclNum) const; + void Clear(); +}; + +//------------------------------------------------------------------------ +// AliasSet: +// Represents a set of reads and writes for the purposes of alias +// analysis. This type partitions storage into two categories: +// lclVars and addressable locations. The definition of the former is +// intuitive. The latter is the union of the set of address-exposed +// lclVars with the set of all other memory locations. Any memory +// access is assumed to alias any other memory access. +// +class AliasSet final +{ + LclVarSet m_lclVarReads; + LclVarSet m_lclVarWrites; + + bool m_readsAddressableLocation; + bool m_writesAddressableLocation; + +public: + //------------------------------------------------------------------------ + // AliasSet::NodeInfo: + // Represents basic alias information for a single IR node. + // + class NodeInfo final + { + enum : unsigned + { + ALIAS_NONE = 0x0, + ALIAS_READS_ADDRESSABLE_LOCATION = 0x1, + ALIAS_WRITES_ADDRESSABLE_LOCATION = 0x2, + ALIAS_READS_LCL_VAR = 0x4, + ALIAS_WRITES_LCL_VAR = 0x8 + }; + + Compiler* m_compiler; + GenTree* m_node; + unsigned m_flags; + unsigned m_lclNum; + + public: + NodeInfo(Compiler* compiler, GenTree* node); + + inline Compiler* TheCompiler() const + { + return m_compiler; + } + + inline GenTree* Node() const + { + return m_node; + } + + inline bool ReadsAddressableLocation() const + { + return (m_flags & ALIAS_READS_ADDRESSABLE_LOCATION) != 0; + } + + inline bool WritesAddressableLocation() const + { + return (m_flags & ALIAS_WRITES_ADDRESSABLE_LOCATION) != 0; + } + + inline bool IsLclVarRead() const + { + return (m_flags & ALIAS_READS_LCL_VAR) != 0; + } + + inline bool IsLclVarWrite() const + { + return (m_flags & ALIAS_WRITES_LCL_VAR) != 0; + } + + inline unsigned LclNum() const + { + assert(IsLclVarRead() || IsLclVarWrite()); + return m_lclNum; + } + + inline bool WritesAnyLocation() const + { + return (m_flags & (ALIAS_WRITES_ADDRESSABLE_LOCATION | ALIAS_WRITES_LCL_VAR)) != 0; + } + }; + + AliasSet(); + + inline bool WritesAnyLocation() const + { + return m_writesAddressableLocation || !m_lclVarWrites.IsEmpty(); + } + + void AddNode(Compiler* compiler, GenTree* node); + bool InterferesWith(const AliasSet& other) const; + bool InterferesWith(const NodeInfo& node) const; + void Clear(); +}; + +//------------------------------------------------------------------------ +// SideEffectSet: +// Represents a set of side effects for the purposes of analyzing code +// motion. +// +class SideEffectSet final +{ + unsigned m_sideEffectFlags; // A mask of GTF_* flags that represents exceptional and barrier side effects. + AliasSet m_aliasSet; // An AliasSet that represents read and write side effects. + + template + bool InterferesWith(unsigned otherSideEffectFlags, const TOtherAliasInfo& otherAliasInfo, bool strict) const; + +public: + SideEffectSet(); + SideEffectSet(Compiler* compiler, GenTree* node); + + void AddNode(Compiler* compiler, GenTree* node); + bool InterferesWith(const SideEffectSet& other, bool strict) const; + bool InterferesWith(Compiler* compiler, GenTree* node, bool strict) const; + void Clear(); +}; + +#endif // _SIDEEFFECTS_H_ diff --git a/src/jit/simdcodegenxarch.cpp b/src/jit/simdcodegenxarch.cpp index 14c4493f07e8..702f967aad2d 100644 --- a/src/jit/simdcodegenxarch.cpp +++ b/src/jit/simdcodegenxarch.cpp @@ -20,6 +20,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #ifdef _TARGET_AMD64_ #include "emit.h" #include "codegen.h" +#include "sideeffects.h" #include "lower.h" #include "gcinfo.h" #include "gcinfoencoder.h"