diff --git a/include/swift/SIL/PrunedLiveness.h b/include/swift/SIL/PrunedLiveness.h index 2ec078beb6ebf..95b05aabaf4ea 100644 --- a/include/swift/SIL/PrunedLiveness.h +++ b/include/swift/SIL/PrunedLiveness.h @@ -590,6 +590,8 @@ struct PrunedLivenessBoundary { /// /// bool isDef(SILInstruction *inst) const /// +/// bool isDef(SILArgument *arg) const +/// /// bool isDefBlock(SILBasicBlock *block) const /// template @@ -611,6 +613,15 @@ class PrunedLiveRange : public PrunedLiveness { ValueSet &visited, SILValue value); + bool isInstructionLive(SILInstruction *instruction, bool liveOut) const; + bool isAvailableOut(SILBasicBlock *block, DeadEndBlocks &deadEndBlocks) const; + bool isInstructionAvailable(SILInstruction *user, + DeadEndBlocks &deadEndBlocks) const; + /// Whether \p user is within the boundary extended from live regions into + /// dead-end regions up to the availability boundary. + bool isWithinExtendedBoundary(SILInstruction *user, + DeadEndBlocks *deadEndBlocks) const; + public: /// Add \p inst to liveness which uses the def as indicated by \p usage. void updateForUse(SILInstruction *inst, LifetimeEnding usage); @@ -735,6 +746,8 @@ class SSAPrunedLiveness : public PrunedLiveRange { bool isDef(SILInstruction *inst) const { return inst == defInst; } + bool isDef(SILArgument *arg) const { return def == arg; } + bool isDefBlock(SILBasicBlock *block) const { return def->getParentBlock() == block; } diff --git a/include/swift/SILOptimizer/Utils/CanonicalizeBorrowScope.h b/include/swift/SILOptimizer/Utils/CanonicalizeBorrowScope.h index 92c5438e71dfd..c683973489e57 100644 --- a/include/swift/SILOptimizer/Utils/CanonicalizeBorrowScope.h +++ b/include/swift/SILOptimizer/Utils/CanonicalizeBorrowScope.h @@ -168,12 +168,6 @@ bool shrinkBorrowScope( MoveValueInst *foldDestroysOfCopiedLexicalBorrow(BeginBorrowInst *bbi, DominanceInfo &dominanceTree, InstructionDeleter &deleter); - -bool hoistDestroysOfOwnedLexicalValue(SILValue const value, - SILFunction &function, - InstructionDeleter &deleter, - BasicCalleeAnalysis *calleeAnalysis); - } // namespace swift #endif // SWIFT_SILOPTIMIZER_UTILS_CANONICALIZEBORROWSCOPES_H diff --git a/include/swift/SILOptimizer/Utils/CanonicalizeOSSALifetime.h b/include/swift/SILOptimizer/Utils/CanonicalizeOSSALifetime.h index a5d13cb6a1c2d..8753ba679f971 100644 --- a/include/swift/SILOptimizer/Utils/CanonicalizeOSSALifetime.h +++ b/include/swift/SILOptimizer/Utils/CanonicalizeOSSALifetime.h @@ -100,6 +100,7 @@ #include "swift/Basic/SmallPtrSetVector.h" #include "swift/SIL/PrunedLiveness.h" #include "swift/SIL/SILInstruction.h" +#include "swift/SILOptimizer/Analysis/DeadEndBlocksAnalysis.h" #include "swift/SILOptimizer/Analysis/DominanceAnalysis.h" #include "swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h" #include "swift/SILOptimizer/Utils/InstructionDeleter.h" @@ -247,6 +248,8 @@ class CanonicalizeOSSALifetime final { // extendLivenessThroughOverlappingAccess is invoked. NonLocalAccessBlocks *accessBlocks = nullptr; + DeadEndBlocksAnalysis *deadEndBlocksAnalysis; + DominanceInfo *domTree = nullptr; BasicCalleeAnalysis *calleeAnalysis; @@ -326,15 +329,14 @@ class CanonicalizeOSSALifetime final { } }; - CanonicalizeOSSALifetime(PruneDebugInsts_t pruneDebugMode, - MaximizeLifetime_t maximizeLifetime, - SILFunction *function, - NonLocalAccessBlockAnalysis *accessBlockAnalysis, - DominanceInfo *domTree, - BasicCalleeAnalysis *calleeAnalysis, - InstructionDeleter &deleter) + CanonicalizeOSSALifetime( + PruneDebugInsts_t pruneDebugMode, MaximizeLifetime_t maximizeLifetime, + SILFunction *function, NonLocalAccessBlockAnalysis *accessBlockAnalysis, + DeadEndBlocksAnalysis *deadEndBlocksAnalysis, DominanceInfo *domTree, + BasicCalleeAnalysis *calleeAnalysis, InstructionDeleter &deleter) : pruneDebugMode(pruneDebugMode), maximizeLifetime(maximizeLifetime), - accessBlockAnalysis(accessBlockAnalysis), domTree(domTree), + accessBlockAnalysis(accessBlockAnalysis), + deadEndBlocksAnalysis(deadEndBlocksAnalysis), domTree(domTree), calleeAnalysis(calleeAnalysis), deleter(deleter) {} SILValue getCurrentDef() const { return currentDef; } @@ -342,11 +344,7 @@ class CanonicalizeOSSALifetime final { void initializeLiveness(SILValue def, ArrayRef lexicalLifetimeEnds) { assert(consumingBlocks.empty() && debugValues.empty()); - // Clear the cached analysis pointer just in case the client invalidates the - // analysis, freeing its memory. - accessBlocks = nullptr; - consumes.clear(); - destroys.clear(); + clear(); currentDef = def; currentLexicalLifetimeEnds = lexicalLifetimeEnds; @@ -358,9 +356,18 @@ class CanonicalizeOSSALifetime final { } void clear() { + // Clear the access blocks analysis pointer in case the client invalidates + // the analysis. If the client did, the analysis will be recomputed in + // extendLivenessThroughOverlappingAccess; if it didn't, the analysis + // pointer will just be set back to its old value when the analysis' cache + // is consulted in extendLivenessThroughOverlappingAccess. + accessBlocks = nullptr; + consumingBlocks.clear(); debugValues.clear(); discoveredBlocks.clear(); + consumes.clear(); + destroys.clear(); } /// Top-Level API: rewrites copies and destroys within \p def's extended @@ -472,7 +479,7 @@ class CanonicalizeOSSALifetime final { void findExtendedBoundary(PrunedLivenessBoundary const &originalBoundary, PrunedLivenessBoundary &boundary); - void findDestroysOutsideBoundary(SmallVectorImpl &destroys); + void extendLivenessToDeadEnds(); void extendLivenessToDeinitBarriers(); void extendUnconsumedLiveness(PrunedLivenessBoundary const &boundary); @@ -481,9 +488,10 @@ class CanonicalizeOSSALifetime final { llvm::function_ref visitor); - void insertDestroysOnBoundary(PrunedLivenessBoundary const &boundary); + void insertDestroysOnBoundary(PrunedLivenessBoundary const &boundary, + SmallVectorImpl &destroys); - void rewriteCopies(); + void rewriteCopies(SmallVectorImpl const &destroys); }; } // end namespace swift diff --git a/lib/SIL/Utils/PrunedLiveness.cpp b/lib/SIL/Utils/PrunedLiveness.cpp index 1fa66de87b5ca..be657ea31d71d 100644 --- a/lib/SIL/Utils/PrunedLiveness.cpp +++ b/lib/SIL/Utils/PrunedLiveness.cpp @@ -515,6 +515,13 @@ bool PrunedLiveRange::isWithinBoundary( if (isLive && !asImpl().isDefBlock(block)) return true; + return isInstructionLive(inst, isLive); +} + +template +bool PrunedLiveRange::isInstructionLive(SILInstruction *inst, + bool isLive) const { + auto *block = inst->getParent(); // Check if instruction is between a last use and a definition for (SILInstruction &it : llvm::reverse(*block)) { // the def itself is not within the boundary, so cancel liveness before @@ -532,53 +539,259 @@ bool PrunedLiveRange::isWithinBoundary( llvm_unreachable("instruction must be in its parent block"); } -/// Whether \p parent is a dead (reported to be dead by `liveBlocks`), dead-end -/// (such as an infinite loop) block within the availability boundary (where -/// the value has not been consumed). -static bool checkDeadEnd(SILBasicBlock *parent, DeadEndBlocks *deadEndBlocks, - PrunedLiveBlocks const &liveBlocks) { +template +bool PrunedLiveRange::isAvailableOut( + SILBasicBlock *block, DeadEndBlocks &deadEndBlocks) const { + assert(getBlockLiveness(block) == PrunedLiveBlocks::LiveWithin); + assert(deadEndBlocks.isDeadEnd(block)); + for (SILInstruction &inst : llvm::reverse(*block)) { + if (asImpl().isDef(&inst)) { + return true; + } + switch (isInterestingUser(&inst)) { + case PrunedLiveness::NonUser: + continue; + case PrunedLiveness::NonLifetimeEndingUse: + return true; + case PrunedLiveness::LifetimeEndingUse: + return false; + } + } + assert(asImpl().isDefBlock(block)); + assert(llvm::any_of(block->getArguments(), [this](SILArgument *arg) { + return asImpl().isDef(arg); + })); + return true; +} + +template +bool PrunedLiveRange::isInstructionAvailable( + SILInstruction *user, DeadEndBlocks &deadEndBlocks) const { + auto *parent = user->getParent(); + assert(getBlockLiveness(parent) == PrunedLiveBlocks::LiveWithin); + assert(deadEndBlocks.isDeadEnd(parent)); + return isInstructionLive(user, isAvailableOut(parent, deadEndBlocks)); +} + +template +bool PrunedLiveRange::isWithinExtendedBoundary( + SILInstruction *inst, DeadEndBlocks *deadEndBlocks) const { + // A value has a pruned live region, a live region and an available region. + // (Note: PrunedLiveness does not distinguish between the pruned live region + // and the live region; the pruned live region coincides with the live region + // whenever consuming uses are considered.) This method refers to a FOURTH + // region: the "extended region" which MAY be different from the others. + // (Terminological note: this isn't intended to gain regular usage, hence its + // lack of specificity.) + // + // Before _defining_ the extended region, consider the following example: + // + // def = ... + // inst_1 + // use %def // added to pruned liveness + // inst_2 + // cond_br %c1, die, normal + // die: + // inst_3 + // unreachable + // normal: + // inst_4 + // destroy %def // NOT added to pruned liveness + // inst_5 + // + // This table describes which regions the `inst_i`s are in: + // +------+----+------+--------+---------+ + // | |live|pruned|extended|available| + // +------+----+------+--------+---------+ + // |inst_1| yes| yes | yes | yes | + // +------+----+------+--------+---------+ + // |inst_2| yes| no | yes | yes | + // +------+----+------+--------+---------+ + // |inst_3| no | no | yes | yes | + // +------+----+------+--------+---------+ + // |inst_4| yes| no | no | yes | + // +------+----+------+--------+---------+ + // |inst_5| no | no | no | no | + // +------+----+------+--------+---------+ + // + // This example demonstrates that + // pruned live ≠ extended ≠ available + // and indicates the fact that + // pruned live ⊆ extended ⊆ available + // + // The "extended region" is the pruned live region availability-extended into + // dead-end regions. In more detail, it's obtained by (1) unioning the + // dead-end regions adjacent to the pruned live region (the portions of those + // adjacent dead-end regions which are forward reachable from the pruned live + // region) and (2) intersecting the result with the availability region. + // + // That this region is of interest is another result of lacking complete + // OSSA lifetimes. + + if (asImpl().isWithinBoundary(inst)) { + // The extended region is a superset of the pruned live region. + return true; + } + if (!deadEndBlocks) { + // Without knowledge of the dead-end region, the extended region can't be + // determined. It could, of course, be rediscovered here, but that would + // be silly; instead, allowing a nullable pointer provides a mechanism for + // the client to indicate what invariants hold. Specifically, omitting + // dead-end blocks is equivalent to asserting that lifetimes are complete. return false; } + SILBasicBlock *parent = inst->getParent(); if (!deadEndBlocks->isDeadEnd(parent)) { + // The extended region intersected with the non-dead-end region is equal to + // the pruned live region. return false; } - if (liveBlocks.getBlockLiveness(parent) != PrunedLiveBlocks::Dead) { + switch (liveBlocks.getBlockLiveness(parent)) { + case PrunedLiveBlocks::Dead: + break; + case PrunedLiveBlocks::LiveWithin: + // Dead defs may result in LiveWithin but AvailableOut blocks. + return isInstructionAvailable(inst, *deadEndBlocks); + case PrunedLiveBlocks::LiveOut: + // The instruction is not within the boundary, but its parent is LiveOut; + // therefore it must be a def block. + assert(asImpl().isDefBlock(parent)); + + // Where within the block might the instruction be? + // - before the first def: return false (outside the extended region). + // - between a def and a use: unreachable (withinBoundary would have + // returned true). + // - between a def and another def: unreachable (withinBoundary would have + // returned true) + // - between a use and a def: return false (outside the extended region). + // - after the final def: unreachable (withinBoundary would have returned + // true) return false; } - // Check whether the value is available in `parent` (i.e. not consumed on any - // path to it): + // Check whether `parent` is in the extended region: walk backwards within + // the dead portion of the dead-end region up _through_ the first block which + // is either not dead or not dead-end. + // + // During the walk, if ANY reached block satisfies one of + // (1) dead-end, LiveWithin, !AvailableOut + // (2) NOT dead-end, NOT LiveOut + // then the `parent` is not in the extended region. // - // Search backward until LiveOut or LiveWithin blocks are reached. - // (1) If ALL the reached blocks are LiveOut, then `parent` IS within the - // availability boundary. - // (2) If ANY reached block is LiveWithin, the value was consumed in that - // reached block, preventing the value from being available at `parent`, - // so `parent` is NOT within the availability boundary. + // Otherwise, ALL reached blocks satisfied one of the following: + // (a) dead-end, Dead + // (b) dead-end, LiveWithin, AvailableOut + // (b) MAYBE dead-end, LiveOut + // In this case, `parent` is in the extended region. BasicBlockWorklist worklist(parent->getFunction()); worklist.push(parent); while (auto *block = worklist.pop()) { auto isLive = liveBlocks.getBlockLiveness(block); + if (!deadEndBlocks->isDeadEnd(block)) { + // The first block beyond the dead-end region has been reached. + if (isLive != PrunedLiveBlocks::LiveOut) { + // Cases (2) above. + return false; + } + // Stop walking. (No longer in the dead portion of the dead-end region.) + continue; + } switch (isLive) { - case PrunedLiveBlocks::Dead: { - // Availability is unchanged; continue the backwards walk. + case PrunedLiveBlocks::Dead: + // Still within the dead portion of the dead-end region. Keep walking. for (auto *predecessor : block->getPredecessorBlocks()) { worklist.pushIfNotVisited(predecessor); } - break; - } + continue; case PrunedLiveBlocks::LiveWithin: - // Availability ended in this block. Some path to `parent` consumed the - // value. Case (2) above. - return false; + // Availability may have ended in this block. Check whether the block is + // "AvailableOut". + if (!isAvailableOut(block, *deadEndBlocks)) { + // Case (1) above. + return false; + } + // Stop walking. (No longer in the dead portion of the dead-end region.) + continue; case PrunedLiveBlocks::LiveOut: - // Availability continued out of this block. Case (1) above. + // Stop walking. (No longer in the dead portion of the dead-end region.) continue; } } return true; } +namespace swift::test { +// Arguments: +// - string: "def:" +// - SILValue: value to be analyzed +// - string: "liveness-uses:" +// - variadic list of - SILInstruction: user to pass to updateForUse +// - string: non-ending/ending/non-use +// - string: "uses:" +// - variadic list of - SILInstruction: the instruction to pass to +// areUsesWithinBoundary Dumps: +// - true/false +static FunctionTest SSAPrunedLiveness__areUsesWithinBoundary( + "SSAPrunedLiveness__areUsesWithinBoundary", + [](auto &function, auto &arguments, auto &test) { + SmallVector discoveredBlocks; + SSAPrunedLiveness liveness(&function, &discoveredBlocks); + + llvm::outs() << "SSAPrunedLiveness:\n"; + + if (arguments.takeString() != "def:") { + llvm::report_fatal_error("test expects the 'def:' label\n"); + } + auto def = arguments.takeValue(); + liveness.initializeDef(def); + llvm::outs() << "\tdef: " << def; + if (arguments.takeString() != "liveness-uses:") { + llvm::report_fatal_error("test expects the 'def:' label\n"); + } + llvm::outs() << "\tuses:\n"; + while (true) { + auto argument = arguments.takeArgument(); + if (isa(argument)) { + auto string = cast(argument); + if (string.getValue() != "uses:") { + llvm::report_fatal_error("test expects the 'inst:' label\n"); + } + break; + } + auto *instruction = cast(argument).getValue(); + auto string = arguments.takeString(); + PrunedLiveness::LifetimeEnding::Value kind = + llvm::StringSwitch(string) + .Case("non-ending", + PrunedLiveness::LifetimeEnding::Value::NonEnding) + .Case("ending", PrunedLiveness::LifetimeEnding::Value::Ending) + .Case("non-use", PrunedLiveness::LifetimeEnding::Value::NonUse); + + llvm::outs() << "\t\t" << string << " " << *instruction; + liveness.updateForUse(instruction, kind); + } + liveness.print(llvm::outs()); + + PrunedLivenessBoundary boundary; + liveness.computeBoundary(boundary); + boundary.print(llvm::outs()); + + llvm::outs() << "\noperands:\n"; + SmallVector operands; + while (arguments.hasUntaken()) { + auto *operand = arguments.takeOperand(); + operands.push_back(operand); + operand->print(llvm::outs()); + } + + auto result = + liveness.areUsesWithinBoundary(operands, test.getDeadEndBlocks()); + + llvm::outs() << "RESULT: " << StringRef(result ? "true" : "false") + << "\n"; + }); +} // end namespace swift::test + template bool PrunedLiveRange::areUsesWithinBoundary( ArrayRef uses, DeadEndBlocks *deadEndBlocks) const { @@ -586,8 +799,7 @@ bool PrunedLiveRange::areUsesWithinBoundary( for (auto *use : uses) { auto *user = use->getUser(); - if (!asImpl().isWithinBoundary(user) && - !checkDeadEnd(user->getParent(), deadEndBlocks, liveBlocks)) + if (!isWithinExtendedBoundary(user, deadEndBlocks)) return false; } return true; @@ -600,8 +812,7 @@ bool PrunedLiveRange::areUsesOutsideBoundary( for (auto *use : uses) { auto *user = use->getUser(); - if (asImpl().isWithinBoundary(user) || - checkDeadEnd(user->getParent(), deadEndBlocks, liveBlocks)) + if (isWithinExtendedBoundary(user, deadEndBlocks)) return false; } return true; diff --git a/lib/SILOptimizer/Mandatory/ConsumeOperatorCopyableValuesChecker.cpp b/lib/SILOptimizer/Mandatory/ConsumeOperatorCopyableValuesChecker.cpp index 2577697120a99..6af0717f70914 100644 --- a/lib/SILOptimizer/Mandatory/ConsumeOperatorCopyableValuesChecker.cpp +++ b/lib/SILOptimizer/Mandatory/ConsumeOperatorCopyableValuesChecker.cpp @@ -222,14 +222,15 @@ struct ConsumeOperatorCopyableValuesChecker { InstructionDeleter deleter; CanonicalizeOSSALifetime canonicalizer; - ConsumeOperatorCopyableValuesChecker(SILFunction *fn, - DominanceInfo *dominance, - BasicCalleeAnalysis *calleeAnalysis) + ConsumeOperatorCopyableValuesChecker( + SILFunction *fn, DominanceInfo *dominance, + BasicCalleeAnalysis *calleeAnalysis, + DeadEndBlocksAnalysis *deadEndBlocksAnalysis) : fn(fn), dominance(dominance), canonicalizer(DontPruneDebugInsts, MaximizeLifetime_t(!fn->shouldOptimize()), fn, - /*accessBlockAnalysis=*/nullptr, dominance, - calleeAnalysis, deleter) {} + /*accessBlockAnalysis=*/nullptr, deadEndBlocksAnalysis, + dominance, calleeAnalysis, deleter) {} bool check(); @@ -588,8 +589,9 @@ class ConsumeOperatorCopyableValuesCheckerPass : public SILFunctionTransform { auto *dominanceAnalysis = getAnalysis(); auto *dominance = dominanceAnalysis->get(fn); auto *calleeAnalysis = getAnalysis(); - ConsumeOperatorCopyableValuesChecker checker(getFunction(), dominance, - calleeAnalysis); + auto *deadEndBlocksAnalysis = getAnalysis(); + ConsumeOperatorCopyableValuesChecker checker( + getFunction(), dominance, calleeAnalysis, deadEndBlocksAnalysis); auto *loopAnalysis = getAnalysis(); if (checker.check()) { diff --git a/lib/SILOptimizer/Mandatory/MoveOnlyAddressCheckerUtils.cpp b/lib/SILOptimizer/Mandatory/MoveOnlyAddressCheckerUtils.cpp index c05de26c15c65..93dd2ce07279e 100644 --- a/lib/SILOptimizer/Mandatory/MoveOnlyAddressCheckerUtils.cpp +++ b/lib/SILOptimizer/Mandatory/MoveOnlyAddressCheckerUtils.cpp @@ -1490,7 +1490,7 @@ struct MoveOnlyAddressCheckerPImpl { DominanceInfo *domTree, PostOrderAnalysis *poa, DeadEndBlocksAnalysis *deba, borrowtodestructure::IntervalMapAllocator &allocator) - : fn(fn), deleter(), canonicalizer(fn, domTree, deleter), + : fn(fn), deleter(), canonicalizer(fn, domTree, deba, deleter), addressUseState(domTree), diagnosticEmitter(diagnosticEmitter), deba(deba), poa(poa), allocator(allocator) { deleter.setCallbacks(std::move( diff --git a/lib/SILOptimizer/Mandatory/MoveOnlyChecker.cpp b/lib/SILOptimizer/Mandatory/MoveOnlyChecker.cpp index 56087b1d82556..a27ffc9d5398a 100644 --- a/lib/SILOptimizer/Mandatory/MoveOnlyChecker.cpp +++ b/lib/SILOptimizer/Mandatory/MoveOnlyChecker.cpp @@ -115,7 +115,8 @@ void MoveOnlyChecker::checkObjects() { completeObjectLifetimes(moveIntroducersToProcess.getArrayRef()); - MoveOnlyObjectChecker checker{diagnosticEmitter, domTree, poa, allocator}; + MoveOnlyObjectChecker checker{diagnosticEmitter, domTree, deba, poa, + allocator}; madeChange |= checker.check(moveIntroducersToProcess); } diff --git a/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerTester.cpp b/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerTester.cpp index 768aea6c20f75..975bae0dd3748 100644 --- a/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerTester.cpp +++ b/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerTester.cpp @@ -89,6 +89,7 @@ class MoveOnlyObjectCheckerTesterPass : public SILFunctionTransform { auto *dominanceAnalysis = getAnalysis(); DominanceInfo *domTree = dominanceAnalysis->get(fn); auto *poa = getAnalysis(); + auto *deba = getAnalysis(); DiagnosticEmitter diagnosticEmitter(fn); borrowtodestructure::IntervalMapAllocator allocator; @@ -111,7 +112,8 @@ class MoveOnlyObjectCheckerTesterPass : public SILFunctionTransform { << "No move introducers found?! Returning early?!\n"); } else { diagCount = diagnosticEmitter.getDiagnosticCount(); - MoveOnlyObjectChecker checker{diagnosticEmitter, domTree, poa, allocator}; + MoveOnlyObjectChecker checker{diagnosticEmitter, domTree, deba, poa, + allocator}; madeChange |= checker.check(moveIntroducersToProcess); } diff --git a/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerUtils.cpp b/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerUtils.cpp index 20bbed24e1f9f..4f655e3b65a30 100644 --- a/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerUtils.cpp +++ b/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerUtils.cpp @@ -175,7 +175,9 @@ struct MoveOnlyObjectCheckerPImpl { : fn(fn), allocator(allocator), diagnosticEmitter(diagnosticEmitter), moveIntroducersToProcess(moveIntroducersToProcess) {} - void check(DominanceInfo *domTree, PostOrderAnalysis *poa); + void check(DominanceInfo *domTree, + DeadEndBlocksAnalysis *deadEndBlocksAnalysis, + PostOrderAnalysis *poa); bool convertBorrowExtractsToOwnedDestructures( MarkUnresolvedNonCopyableValueInst *mmci, DominanceInfo *domTree, @@ -331,8 +333,9 @@ bool MoveOnlyObjectCheckerPImpl::checkForSameInstMultipleUseErrors( // MARK: Main PImpl Routine //===----------------------------------------------------------------------===// -void MoveOnlyObjectCheckerPImpl::check(DominanceInfo *domTree, - PostOrderAnalysis *poa) { +void MoveOnlyObjectCheckerPImpl::check( + DominanceInfo *domTree, DeadEndBlocksAnalysis *deadEndBlocksAnalysis, + PostOrderAnalysis *poa) { auto callbacks = InstModCallbacks().onDelete([&](SILInstruction *instToDelete) { if (auto *mvi = @@ -341,7 +344,7 @@ void MoveOnlyObjectCheckerPImpl::check(DominanceInfo *domTree, instToDelete->eraseFromParent(); }); InstructionDeleter deleter(std::move(callbacks)); - OSSACanonicalizer canonicalizer(fn, domTree, deleter); + OSSACanonicalizer canonicalizer(fn, domTree, deadEndBlocksAnalysis, deleter); diagnosticEmitter.initCanonicalizer(&canonicalizer); unsigned initialDiagCount = diagnosticEmitter.getDiagnosticCount(); @@ -561,6 +564,6 @@ bool MoveOnlyObjectChecker::check( "Should only call this with actual insts to check?!"); MoveOnlyObjectCheckerPImpl checker(instsToCheck[0]->getFunction(), allocator, diagnosticEmitter, instsToCheck); - checker.check(domTree, poa); + checker.check(domTree, deadEndBlocksAnalysis, poa); return checker.changed; } diff --git a/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerUtils.h b/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerUtils.h index 483693898a003..cefd8551db131 100644 --- a/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerUtils.h +++ b/lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerUtils.h @@ -47,11 +47,12 @@ struct OSSACanonicalizer { CanonicalizeOSSALifetime canonicalizer; OSSACanonicalizer(SILFunction *fn, DominanceInfo *domTree, + DeadEndBlocksAnalysis *deadEndBlocksAnalysis, InstructionDeleter &deleter) : canonicalizer(DontPruneDebugInsts, MaximizeLifetime_t(!fn->shouldOptimize()), fn, - nullptr /*accessBlockAnalysis*/, domTree, - nullptr /*calleeAnalysis*/, deleter) {} + nullptr /*accessBlockAnalysis*/, deadEndBlocksAnalysis, + domTree, nullptr /*calleeAnalysis*/, deleter) {} void clear() { consumingUsesNeedingCopy.clear(); @@ -176,6 +177,7 @@ bool searchForCandidateObjectMarkUnresolvedNonCopyableValueInsts( struct MoveOnlyObjectChecker { DiagnosticEmitter &diagnosticEmitter; DominanceInfo *domTree; + DeadEndBlocksAnalysis *deadEndBlocksAnalysis; PostOrderAnalysis *poa; borrowtodestructure::IntervalMapAllocator &allocator; diff --git a/lib/SILOptimizer/SILCombiner/SILCombine.cpp b/lib/SILOptimizer/SILCombiner/SILCombine.cpp index d77814d1f18bd..2f8a26239f991 100644 --- a/lib/SILOptimizer/SILCombiner/SILCombine.cpp +++ b/lib/SILOptimizer/SILCombiner/SILCombine.cpp @@ -192,8 +192,8 @@ SILCombiner::SILCombiner(SILFunctionTransform *trans, use->set(newValue); Worklist.add(use->getUser()); })), - deadEndBlocks(trans->getFunction()), MadeChange(false), - RemoveCondFails(removeCondFails), + DEBA(trans->getPassManager()->getAnalysis()), + MadeChange(false), RemoveCondFails(removeCondFails), enableCopyPropagation(enableCopyPropagation), Iteration(0), Builder(*trans->getFunction(), &TrackingList), FuncBuilder(*trans), @@ -354,7 +354,7 @@ void SILCombiner::canonicalizeOSSALifetimes(SILInstruction *currentInst) { CanonicalizeOSSALifetime canonicalizer( DontPruneDebugInsts, MaximizeLifetime_t(!parentTransform->getFunction()->shouldOptimize()), - parentTransform->getFunction(), NLABA, domTree, CA, deleter); + parentTransform->getFunction(), NLABA, DEBA, domTree, CA, deleter); CanonicalizeBorrowScope borrowCanonicalizer(parentTransform->getFunction(), deleter); @@ -397,7 +397,7 @@ bool SILCombiner::doOneIteration(SILFunction &F, unsigned Iteration) { // Add reachable instructions to our worklist. addReachableCodeToWorklist(&*F.begin()); - SILCombineCanonicalize scCanonicalize(Worklist, deadEndBlocks); + SILCombineCanonicalize scCanonicalize(Worklist, *DEBA->get(&F)); // Process until we run out of items in our worklist. while (!Worklist.isEmpty()) { diff --git a/lib/SILOptimizer/SILCombiner/SILCombiner.h b/lib/SILOptimizer/SILCombiner/SILCombiner.h index bd228f87d5db6..31a502165759f 100644 --- a/lib/SILOptimizer/SILCombiner/SILCombiner.h +++ b/lib/SILOptimizer/SILCombiner/SILCombiner.h @@ -31,6 +31,7 @@ #include "swift/SIL/SILVisitor.h" #include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h" #include "swift/SILOptimizer/Analysis/ClassHierarchyAnalysis.h" +#include "swift/SILOptimizer/Analysis/DeadEndBlocksAnalysis.h" #include "swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h" #include "swift/SILOptimizer/Analysis/ProtocolConformanceAnalysis.h" #include "swift/SILOptimizer/OptimizerBridging.h" @@ -83,7 +84,7 @@ class SILCombiner : /// A cache of "dead end blocks" through which all paths it is known that the /// program will terminate. This means that we are allowed to leak /// objects. - DeadEndBlocks deadEndBlocks; + DeadEndBlocksAnalysis *DEBA; /// Variable to track if the SILCombiner made any changes. bool MadeChange; diff --git a/lib/SILOptimizer/Transforms/CopyPropagation.cpp b/lib/SILOptimizer/Transforms/CopyPropagation.cpp index 5dab316157c3b..41c356ebd6177 100644 --- a/lib/SILOptimizer/Transforms/CopyPropagation.cpp +++ b/lib/SILOptimizer/Transforms/CopyPropagation.cpp @@ -441,6 +441,8 @@ class CopyPropagation : public SILFunctionTransform { /// The entry point to this function transformation. void run() override; + + void verifyOwnership(); }; } // end anonymous namespace @@ -450,6 +452,7 @@ void CopyPropagation::run() { auto *f = getFunction(); auto *postOrderAnalysis = getAnalysis(); auto *accessBlockAnalysis = getAnalysis(); + auto *deadEndBlocksAnalysis = getAnalysis(); auto *dominanceAnalysis = getAnalysis(); auto *calleeAnalysis = getAnalysis(); DominanceInfo *domTree = dominanceAnalysis->get(f); @@ -495,7 +498,8 @@ void CopyPropagation::run() { // don't need to explicitly check for changes. CanonicalizeOSSALifetime canonicalizer( pruneDebug, MaximizeLifetime_t(!getFunction()->shouldOptimize()), - getFunction(), accessBlockAnalysis, domTree, calleeAnalysis, deleter); + getFunction(), accessBlockAnalysis, deadEndBlocksAnalysis, domTree, + calleeAnalysis, deleter); // NOTE: We assume that the function is in reverse post order so visiting the // blocks and pushing begin_borrows as we see them and then popping them // off the end will result in shrinking inner borrow scopes first. @@ -503,8 +507,9 @@ void CopyPropagation::run() { bool firstRun = true; // Run the sequence of utilities: // - ShrinkBorrowScope - // - CanonicalizeOSSALifetime + // - CanonicalizeOSSALifetime(borrowee) // - LexicalDestroyFolding + // - CanonicalizeOSSALifetime(folded) // at least once and then until each stops making changes. while (true) { SmallVector modifiedCopyValueInsts; @@ -529,8 +534,7 @@ void CopyPropagation::run() { auto folded = foldDestroysOfCopiedLexicalBorrow(bbi, *domTree, deleter); if (!folded) break; - auto hoisted = - hoistDestroysOfOwnedLexicalValue(folded, *f, deleter, calleeAnalysis); + auto hoisted = canonicalizer.canonicalizeValueLifetime(folded); // Keep running even if the new move's destroys can't be hoisted. (void)hoisted; eliminateRedundantMove(folded, deleter, defWorklist); @@ -542,7 +546,7 @@ void CopyPropagation::run() { } for (auto *argument : f->getArguments()) { if (argument->getOwnershipKind() == OwnershipKind::Owned) { - hoistDestroysOfOwnedLexicalValue(argument, *f, deleter, calleeAnalysis); + canonicalizer.canonicalizeValueLifetime(argument); } } deleter.cleanupDeadInstructions(); @@ -635,14 +639,19 @@ void CopyPropagation::run() { invalidateAnalysis(SILAnalysis::InvalidationKind::Instructions); accessBlockAnalysis->unlockInvalidation(); if (f->getModule().getOptions().VerifySILOwnership) { - auto *deBlocksAnalysis = getAnalysis(); - f->verifyOwnership(f->getModule().getOptions().OSSAVerifyComplete - ? nullptr - : deBlocksAnalysis->get(f)); + verifyOwnership(); } } } +void CopyPropagation::verifyOwnership() { + auto *f = getFunction(); + auto *deBlocksAnalysis = getAnalysis(); + f->verifyOwnership(f->getModule().getOptions().OSSAVerifyComplete + ? nullptr + : deBlocksAnalysis->get(f)); +} + // MandatoryCopyPropagation is not currently enabled in the -Onone pipeline // because it may negatively affect the debugging experience. SILTransform *swift::createMandatoryCopyPropagation() { diff --git a/lib/SILOptimizer/Transforms/SILMem2Reg.cpp b/lib/SILOptimizer/Transforms/SILMem2Reg.cpp index de76e446e8843..aa7e168e64c02 100644 --- a/lib/SILOptimizer/Transforms/SILMem2Reg.cpp +++ b/lib/SILOptimizer/Transforms/SILMem2Reg.cpp @@ -2146,7 +2146,8 @@ void MemoryToRegisters::canonicalizeValueLifetimes( } CanonicalizeOSSALifetime canonicalizer( PruneDebugInsts, MaximizeLifetime_t(!f.shouldOptimize()), &f, - accessBlockAnalysis, domInfo, calleeAnalysis, deleter); + accessBlockAnalysis, deadEndBlocksAnalysis, domInfo, calleeAnalysis, + deleter); for (auto value : owned) { if (isa(value) || value->isMarkedAsDeleted()) continue; diff --git a/lib/SILOptimizer/Utils/CMakeLists.txt b/lib/SILOptimizer/Utils/CMakeLists.txt index b82eb71c1263b..c451a7f30955d 100644 --- a/lib/SILOptimizer/Utils/CMakeLists.txt +++ b/lib/SILOptimizer/Utils/CMakeLists.txt @@ -19,7 +19,6 @@ target_sources(swiftSILOptimizer PRIVATE InstOptUtils.cpp KeyPathProjector.cpp LexicalDestroyFolding.cpp - LexicalDestroyHoisting.cpp LoopUtils.cpp OptimizerStatsUtils.cpp PartialApplyCombiner.cpp diff --git a/lib/SILOptimizer/Utils/CanonicalizeOSSALifetime.cpp b/lib/SILOptimizer/Utils/CanonicalizeOSSALifetime.cpp index 8abc9eece29dc..a2f8493a23c84 100644 --- a/lib/SILOptimizer/Utils/CanonicalizeOSSALifetime.cpp +++ b/lib/SILOptimizer/Utils/CanonicalizeOSSALifetime.cpp @@ -252,26 +252,16 @@ bool CanonicalizeOSSALifetime::computeCanonicalLiveness() { return true; } -void CanonicalizeOSSALifetime::findDestroysOutsideBoundary( - SmallVectorImpl &outsideDestroys) { - for (auto destroy : destroys) { - if (liveness->isWithinBoundary(destroy)) - continue; - outsideDestroys.push_back(destroy); - } -} - -void CanonicalizeOSSALifetime::extendLivenessToDeinitBarriers() { - SmallVector outsideDestroys; - findDestroysOutsideBoundary(outsideDestroys); - - // OSSALifetimeCompletion: With complete lifetimes, creating completeLiveness - // and using it to visit unreachable lifetime ends should be deleted. +void CanonicalizeOSSALifetime::extendLivenessToDeadEnds() { + // TODO: OSSALifetimeCompletion: Once lifetimes are always complete, delete + // this method. SmallVector discoveredBlocks(this->discoveredBlocks); SSAPrunedLiveness completeLiveness(*liveness, &discoveredBlocks); - for (auto *end : outsideDestroys) { - completeLiveness.updateForUse(end, /*lifetimeEnding*/ true); + for (auto destroy : destroys) { + if (liveness->isWithinBoundary(destroy)) + continue; + completeLiveness.updateForUse(destroy, /*lifetimeEnding*/ true); } OSSALifetimeCompletion::visitAvailabilityBoundary( @@ -284,21 +274,25 @@ void CanonicalizeOSSALifetime::extendLivenessToDeinitBarriers() { return true; }); }); +} - ArrayRef ends = {}; - SmallVector lexicalEnds; +void CanonicalizeOSSALifetime::extendLivenessToDeinitBarriers() { + SmallVector ends; if (currentLexicalLifetimeEnds.size() > 0) { visitExtendedUnconsumedBoundary( currentLexicalLifetimeEnds, - [&lexicalEnds](auto *instruction, auto lifetimeEnding) { + [&ends](auto *instruction, auto lifetimeEnding) { instruction->visitSubsequentInstructions([&](auto *next) { - lexicalEnds.push_back(next); + ends.push_back(next); return true; }); }); - ends = lexicalEnds; } else { - ends = outsideDestroys; + for (auto destroy : destroys) { + if (destroy->getOperand(0) != getCurrentDef()) + continue; + ends.push_back(destroy); + } } auto *def = getCurrentDef()->getDefiningInstruction(); @@ -947,10 +941,12 @@ void CanonicalizeOSSALifetime::findExtendedBoundary( /// Create a new destroy_value instruction before the specified instruction and /// record it as a final consume. -static void insertDestroyBeforeInstruction(SILInstruction *nextInstruction, - SILValue currentDef, - CanonicalOSSAConsumeInfo &consumes, - InstModCallbacks &callbacks) { +static void +insertDestroyBeforeInstruction(SILInstruction *nextInstruction, + SILValue currentDef, + CanonicalOSSAConsumeInfo &consumes, + SmallVectorImpl &destroys, + InstModCallbacks &callbacks) { // OSSALifetimeCompletion: This conditional clause can be deleted with // complete lifetimes. if (consumes.isUnreachableLifetimeEnd(nextInstruction)) { @@ -982,6 +978,7 @@ static void insertDestroyBeforeInstruction(SILInstruction *nextInstruction, callbacks.createdNewInst(dvi); consumes.recordFinalConsume(dvi); ++NumDestroysGenerated; + destroys.push_back(dvi); } /// Inserts destroys along the boundary where needed and records all final @@ -993,7 +990,8 @@ static void insertDestroyBeforeInstruction(SILInstruction *nextInstruction, /// - The postdominating consumes cannot be within nested loops. /// - Any blocks in nested loops are now marked LiveOut. void CanonicalizeOSSALifetime::insertDestroysOnBoundary( - PrunedLivenessBoundary const &boundary) { + PrunedLivenessBoundary const &boundary, + SmallVectorImpl &newDestroys) { BasicBlockSet seenMergePoints(getCurrentDef()->getFunction()); for (auto *instruction : boundary.lastUsers) { if (destroys.contains(instruction)) { @@ -1015,7 +1013,7 @@ void CanonicalizeOSSALifetime::insertDestroysOnBoundary( } auto *insertionPoint = &*successor->begin(); insertDestroyBeforeInstruction(insertionPoint, getCurrentDef(), - consumes, getCallbacks()); + consumes, newDestroys, getCallbacks()); LLVM_DEBUG(llvm::dbgs() << " Destroy after terminator " << *instruction << " at beginning of "; successor->printID(llvm::dbgs(), false); @@ -1025,7 +1023,7 @@ void CanonicalizeOSSALifetime::insertDestroysOnBoundary( } auto *insertionPoint = instruction->getNextInstruction(); insertDestroyBeforeInstruction(insertionPoint, getCurrentDef(), consumes, - getCallbacks()); + newDestroys, getCallbacks()); LLVM_DEBUG(llvm::dbgs() << " Destroy at last use " << insertionPoint << "\n"); continue; @@ -1034,14 +1032,14 @@ void CanonicalizeOSSALifetime::insertDestroysOnBoundary( for (auto *edgeDestination : boundary.boundaryEdges) { auto *insertionPoint = &*edgeDestination->begin(); insertDestroyBeforeInstruction(insertionPoint, getCurrentDef(), consumes, - getCallbacks()); + newDestroys, getCallbacks()); LLVM_DEBUG(llvm::dbgs() << " Destroy on edge " << edgeDestination << "\n"); } for (auto *def : boundary.deadDefs) { if (auto *arg = dyn_cast(def)) { auto *insertionPoint = &*arg->getParent()->begin(); insertDestroyBeforeInstruction(insertionPoint, getCurrentDef(), consumes, - getCallbacks()); + newDestroys, getCallbacks()); LLVM_DEBUG(llvm::dbgs() << " Destroy after dead def arg " << arg << "\n"); } else { @@ -1049,7 +1047,7 @@ void CanonicalizeOSSALifetime::insertDestroysOnBoundary( auto *insertionPoint = instruction->getNextInstruction(); assert(insertionPoint && "def instruction was a terminator?!"); insertDestroyBeforeInstruction(insertionPoint, getCurrentDef(), consumes, - getCallbacks()); + newDestroys, getCallbacks()); LLVM_DEBUG(llvm::dbgs() << " Destroy after dead def inst " << instruction << "\n"); } @@ -1079,7 +1077,8 @@ void swift::copyLiveUse(Operand *use, InstModCallbacks &instModCallbacks) { /// Revisit the def-use chain of currentDef. Mark unneeded original /// copies and destroys for deletion. Insert new copies for interior uses that /// require ownership of the used operand. -void CanonicalizeOSSALifetime::rewriteCopies() { +void CanonicalizeOSSALifetime::rewriteCopies( + SmallVectorImpl const &newDestroys) { assert(getCurrentDef()->getOwnershipKind() == OwnershipKind::Owned); InstructionSetVector instsToDelete(getCurrentDef()->getFunction()); @@ -1165,18 +1164,23 @@ void CanonicalizeOSSALifetime::rewriteCopies() { assert(!consumes.hasUnclaimedConsumes()); if (pruneDebugMode) { + for (auto *destroy : newDestroys) { + liveness->updateForUse(destroy, /*lifetimeEnding=*/true); + } for (auto *dvi : debugValues) { - if (!liveness->isWithinBoundary(dvi)) { - LLVM_DEBUG(llvm::dbgs() << " Removing debug_value: " << *dvi); - deleter.forceDelete(dvi); + if (liveness->areUsesWithinBoundary( + {&dvi->getOperandRef()}, + deadEndBlocksAnalysis->get(getCurrentDef()->getFunction()))) { + continue; } + LLVM_DEBUG(llvm::dbgs() << " Removing debug_value: " << *dvi); + deleter.forceDelete(dvi); } } // Remove the leftover copy_value and destroy_value instructions. - for (auto iter = instsToDelete.begin(), end = instsToDelete.end(); - iter != end; ++iter) { - deleter.forceDelete(*iter); + for (auto *inst : instsToDelete) { + deleter.forceDelete(inst); } } @@ -1213,6 +1217,7 @@ bool CanonicalizeOSSALifetime::computeLiveness() { return false; } if (respectsDeinitBarriers()) { + extendLivenessToDeadEnds(); extendLivenessToDeinitBarriers(); } if (accessBlockAnalysis) { @@ -1241,13 +1246,13 @@ void CanonicalizeOSSALifetime::rewriteLifetimes() { findExtendedBoundary(originalBoundary, extendedBoundary); } + SmallVector newDestroys; // Step 5: insert destroys and record consumes - insertDestroysOnBoundary(extendedBoundary); + insertDestroysOnBoundary(extendedBoundary, newDestroys); // Step 6: rewrite copies and delete extra destroys - rewriteCopies(); + rewriteCopies(newDestroys); clear(); - consumes.clear(); } /// Canonicalize a single extended owned lifetime. @@ -1289,6 +1294,8 @@ static FunctionTest CanonicalizeOSSALifetimeTest( [](auto &function, auto &arguments, auto &test) { auto *accessBlockAnalysis = test.template getAnalysis(); + auto *deadEndBlocksAnalysis = + test.template getAnalysis(); auto *dominanceAnalysis = test.template getAnalysis(); DominanceInfo *domTree = dominanceAnalysis->get(&function); auto *calleeAnalysis = test.template getAnalysis(); @@ -1298,8 +1305,8 @@ static FunctionTest CanonicalizeOSSALifetimeTest( InstructionDeleter deleter; CanonicalizeOSSALifetime canonicalizer( pruneDebug, maximizeLifetimes, &function, - respectAccessScopes ? accessBlockAnalysis : nullptr, domTree, - calleeAnalysis, deleter); + respectAccessScopes ? accessBlockAnalysis : nullptr, + deadEndBlocksAnalysis, domTree, calleeAnalysis, deleter); auto value = arguments.takeValue(); SmallVector lexicalLifetimeEnds; while (arguments.hasUntaken()) { diff --git a/lib/SILOptimizer/Utils/LexicalDestroyHoisting.cpp b/lib/SILOptimizer/Utils/LexicalDestroyHoisting.cpp deleted file mode 100644 index 93c20d025c217..0000000000000 --- a/lib/SILOptimizer/Utils/LexicalDestroyHoisting.cpp +++ /dev/null @@ -1,432 +0,0 @@ -//=-- LexicalDestroyHoisting.cpp - Hoist destroy_values to deinit barriers. -=// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -/// Hoist destroys of owned lexical values (owned arguments and the results of -/// move_value [lexical] instructions) up to deinit barriers. -//===----------------------------------------------------------------------===// - -#include "swift/AST/Builtins.h" -#include "swift/Basic/Assertions.h" -#include "swift/SIL/MemAccessUtils.h" -#include "swift/SIL/OwnershipUtils.h" -#include "swift/SIL/SILBasicBlock.h" -#include "swift/SIL/SILInstruction.h" -#include "swift/SIL/SILValue.h" -#include "swift/SIL/Test.h" -#include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h" -#include "swift/SILOptimizer/Analysis/Reachability.h" -#include "swift/SILOptimizer/Analysis/VisitBarrierAccessScopes.h" -#include "swift/SILOptimizer/PassManager/Transforms.h" -#include "swift/SILOptimizer/Utils/CanonicalizeBorrowScope.h" -#include "swift/SILOptimizer/Utils/InstOptUtils.h" -#include "swift/SILOptimizer/Utils/InstructionDeleter.h" -#include "llvm/ADT/STLExtras.h" - -#define DEBUG_TYPE "copy-propagation" - -using namespace swift; - -//===----------------------------------------------------------------------===// -// MARK: LexicalDestroyHoisting -//===----------------------------------------------------------------------===// - -namespace LexicalDestroyHoisting { - -/// The environment within which to hoist. -struct Context final { - /// The owned lexical value whose destroys are to be hoisted. - SILValue const &value; - - /// value->getDefiningInstruction() - SILInstruction *const definition; - - SILBasicBlock *defBlock; - - SILFunction &function; - - InstructionDeleter &deleter; - - BasicCalleeAnalysis *calleeAnalysis; - - Context(SILValue const &value, SILFunction &function, - InstructionDeleter &deleter, BasicCalleeAnalysis *calleeAnalysis) - : value(value), definition(value->getDefiningInstruction()), - defBlock(value->getParentBlock()), function(function), deleter(deleter), - calleeAnalysis(calleeAnalysis) { - assert(value->isLexical()); - assert(value->getOwnershipKind() == OwnershipKind::Owned); - } - Context(Context const &) = delete; - Context &operator=(Context const &) = delete; -}; - -/// How %value gets used. -struct Usage final { - /// Instructions which are users of the simple (i.e. not reborrowed) value. - SmallPtrSet users; - // The instructions from which the hoisting starts, the destroy_values. - llvm::SmallSetVector ends; - - Usage(){}; - Usage(Usage const &) = delete; - Usage &operator=(Usage const &) = delete; -}; - -/// Identify users and destroy_values of %value. -/// -/// returns true if all uses were found -/// false otherwise -bool findUsage(Context const &context, Usage &usage) { - SmallVector uses; - if (!findUsesOfSimpleValue(context.value, &uses)) { - // If the value escapes, don't hoist. - return false; - } - for (auto *use : uses) { - // Add the destroy_values to the collection of ends so we can seed the data - // flow and determine whether any were reused. They aren't uses over which - // we can't hoist though. - auto dv = dyn_cast(use->getUser()); - if (dv && dv->getOperand() == context.value) { - usage.ends.insert(use->getUser()); - } else { - usage.users.insert(use->getUser()); - } - } - return true; -} - -/// How destroy_value hoisting is obstructed. -struct DeinitBarriers final { - /// Instructions above which destroy_values cannot be hoisted. - SmallVector instructions; - - /// Blocks one of whose phis is a barrier and consequently out of which - /// destroy_values cannot be hoisted. - SmallVector phis; - - SmallVector blocks; - - DeinitBarriers(Context &context) {} - DeinitBarriers(DeinitBarriers const &) = delete; - DeinitBarriers &operator=(DeinitBarriers const &) = delete; -}; - -class BarrierAccessScopeFinder; - -/// Works backwards from the current location of destroy_values to the earliest -/// place they can be hoisted to. -/// -/// Implements IterativeBackwardReachability::Effects -/// Implements IterativeBackwardReachability::bindBarriers::Visitor -/// Implements VisitBarrierAccessScopes::Effects -class Dataflow final { - using Reachability = IterativeBackwardReachability; - using Effect = Reachability::Effect; - Context const &context; - Usage const &uses; - DeinitBarriers &barriers; - Reachability::Result result; - Reachability reachability; - SmallPtrSet barrierAccessScopes; - - enum class Classification { Barrier, Other }; - -public: - Dataflow(Context const &context, Usage const &uses, DeinitBarriers &barriers) - : context(context), uses(uses), barriers(barriers), - result(&context.function), - reachability(Reachability::untilInitialBlock( - &context.function, context.defBlock, *this, result)) {} - Dataflow(Dataflow const &) = delete; - Dataflow &operator=(Dataflow const &) = delete; - - void run(); - -private: - friend Reachability; - friend class BarrierAccessScopeFinder; - friend class VisitBarrierAccessScopes; - - Classification classifyInstruction(SILInstruction *); - - bool classificationIsBarrier(Classification); - - /// IterativeBackwardReachability::Effects - /// VisitBarrierAccessScopes::Effects - - auto gens() { return uses.ends; } - - Effect effectForInstruction(SILInstruction *); - Effect effectForPhi(SILBasicBlock *); - - /// VisitBarrierAccessScopes::Effects - - auto localGens() { return result.localGens; } - - bool isLocalGen(SILInstruction *instruction) { - return result.localGens.contains(instruction); - } - - /// IterativeBackwardReachability::bindBarriers::Visitor - - void visitBarrierInstruction(SILInstruction *instruction) { - barriers.instructions.push_back(instruction); - } - - void visitBarrierPhi(SILBasicBlock *block) { barriers.phis.push_back(block); } - - void visitBarrierBlock(SILBasicBlock *block) { - barriers.blocks.push_back(block); - } - - void visitInitialBlock(SILBasicBlock *block) { - barriers.blocks.push_back(block); - } -}; - -Dataflow::Classification -Dataflow::classifyInstruction(SILInstruction *instruction) { - if (instruction == context.definition) { - return Classification::Barrier; - } - if (uses.users.contains(instruction)) { - return Classification::Barrier; - } - if (auto *eai = dyn_cast(instruction)) { - return barrierAccessScopes.contains(eai->getBeginAccess()) - ? Classification::Barrier - : Classification::Other; - } - if (isDeinitBarrier(instruction, context.calleeAnalysis)) { - return Classification::Barrier; - } - return Classification::Other; -} - -bool Dataflow::classificationIsBarrier(Classification classification) { - switch (classification) { - case Classification::Barrier: - return true; - case Classification::Other: - return false; - } - llvm_unreachable("exhaustive switch not exhaustive?!"); -} - -Dataflow::Effect Dataflow::effectForInstruction(SILInstruction *instruction) { - if (uses.ends.contains(instruction)) - return Effect::Gen(); - auto classification = classifyInstruction(instruction); - return classificationIsBarrier(classification) ? Effect::Kill() - : Effect::NoEffect(); -} - -Dataflow::Effect Dataflow::effectForPhi(SILBasicBlock *block) { - assert(llvm::all_of(block->getArguments(), - [&](auto argument) { return PhiValue(argument); })); - - bool isBarrier = - llvm::any_of(block->getPredecessorBlocks(), [&](auto *predecessor) { - return classificationIsBarrier( - classifyInstruction(predecessor->getTerminator())); - }); - return isBarrier ? Effect::Kill() : Effect::NoEffect(); -} - -/// Finds end_access instructions which are barriers to hoisting because the -/// access scopes they contain barriers to hoisting. Hoisting destroy_values -/// into such access scopes could introduce exclusivity violations. -/// -/// Implements BarrierAccessScopeFinder::Visitor -class BarrierAccessScopeFinder final { - using Impl = VisitBarrierAccessScopes; - Impl impl; - Dataflow &dataflow; - -public: - BarrierAccessScopeFinder(Context const &context, Dataflow &dataflow) - : impl(&context.function, dataflow, *this), dataflow(dataflow) {} - - void find() { impl.visit(); } - -private: - friend Impl; - - bool isInRegion(SILBasicBlock *block) { - return dataflow.result.discoveredBlocks.contains(block); - } - - void visitBarrierAccessScope(BeginAccessInst *bai) { - dataflow.barrierAccessScopes.insert(bai); - for (auto *eai : bai->getEndAccesses()) { - dataflow.reachability.addKill(eai); - } - } -}; - -void Dataflow::run() { - reachability.initialize(); - BarrierAccessScopeFinder finder(context, *this); - finder.find(); - reachability.solve(); - reachability.findBarriers(*this); -} - -/// Hoist the destroy_values of %value. -class Rewriter final { - Context &context; - Usage const &uses; - DeinitBarriers const &barriers; - - /// The destroy_value instructions for this owned lexical value that existed - /// before LexicalDestroyHoisting ran and which were not modified. - llvm::SmallPtrSet reusedDestroyValueInsts; - -public: - Rewriter(Context &context, Usage const &uses, DeinitBarriers const &barriers) - : context(context), uses(uses), barriers(barriers) {} - Rewriter(Rewriter const &) = delete; - Rewriter &operator=(Rewriter const &) = delete; - - bool run(); - -private: - bool createDestroyValue(SILInstruction *insertionPoint); -}; - -bool Rewriter::run() { - bool madeChange = false; - - // Add destroy_values for phi barrier boundaries. - // - // A block is a phi barrier iff any of its predecessors' terminators get - // classified as barriers. - for (auto *block : barriers.phis) { - madeChange |= createDestroyValue(&block->front()); - } - - // Add destroy_values for barrier boundaries. - // - // Insert destroy_values after every non-terminator barrier. - // - // For terminator barriers, add destroy_values at the beginning of the - // successor blocks. In order to reach a terminator and classify it as a - // barrier, all of a block P's successors B had reachable beginnings. If any - // of them didn't, then BackwardReachability::meetOverSuccessors would never - // have returned true for P, so none of its instructions would ever have been - // classified (except for via checkReachablePhiBarrier, which doesn't record - // terminator barriers). - for (auto instruction : barriers.instructions) { - if (auto *terminator = dyn_cast(instruction)) { - auto successors = terminator->getParentBlock()->getSuccessorBlocks(); - for (auto *successor : successors) { - madeChange |= createDestroyValue(&successor->front()); - } - } else { - auto *next = instruction->getNextInstruction(); - assert(next); - madeChange |= createDestroyValue(next); - } - } - - // Add destroy_values for control-flow boundaries. - // - // Insert destroy_values at the beginning of blocks which were preceded by a - // control flow branch (and which, thanks to the lack of critical edges, - // don't have multiple predecessors) whose end was not reachable (because - // reachability was not able to make it to the top of some other successor). - // - // In other words, a control flow boundary is the target edge from a block B - // to its single predecessor P not all of whose successors S in succ(P) had - // reachable beginnings. We witness that fact about P's successors by way of - // P not having a reachable end--see BackwardReachability::meetOverSuccessors. - // - // control-flow-boundary(B) := beginning-reachable(B) && !end-reachable(P) - for (auto *block : barriers.blocks) { - madeChange |= createDestroyValue(&block->front()); - } - - if (madeChange) { - // Remove all the original destroy_values instructions. - for (auto *end : uses.ends) { - if (reusedDestroyValueInsts.contains(end)) { - continue; - } - context.deleter.forceDelete(end); - } - } - - return madeChange; -} - -bool Rewriter::createDestroyValue(SILInstruction *insertionPoint) { - if (auto *ebi = dyn_cast(insertionPoint)) { - if (llvm::find(uses.ends, insertionPoint) != uses.ends.end()) { - reusedDestroyValueInsts.insert(insertionPoint); - return false; - } - } - auto builder = SILBuilderWithScope(insertionPoint); - builder.createDestroyValue( - RegularLocation::getAutoGeneratedLocation(insertionPoint->getLoc()), - context.value); - return true; -} - -bool run(Context &context) { - Usage usage; - if (!findUsage(context, usage)) - return false; - - DeinitBarriers barriers(context); - Dataflow flow(context, usage, barriers); - flow.run(); - - Rewriter rewriter(context, usage, barriers); - - return rewriter.run(); -} -} // end namespace LexicalDestroyHoisting - -bool swift::hoistDestroysOfOwnedLexicalValue( - SILValue const value, SILFunction &function, InstructionDeleter &deleter, - BasicCalleeAnalysis *calleeAnalysis) { - if (!value->isLexical()) - return false; - if (value->getOwnershipKind() != OwnershipKind::Owned) - return false; - LexicalDestroyHoisting::Context context(value, function, deleter, - calleeAnalysis); - return LexicalDestroyHoisting::run(context); -} - -namespace swift::test { -// Arguments: -// - bool: pruneDebug -// - bool: maximizeLifetimes -// - bool: "respectAccessScopes", whether to contract lifetimes to end within -// access scopes which they previously enclosed but can't be hoisted -// before -// - SILValue: value to canonicalize -// Dumps: -// - function after value canonicalization -static FunctionTest LexicalDestroyHoistingTest( - "lexical_destroy_hoisting", - [](auto &function, auto &arguments, auto &test) { - auto *calleeAnalysis = test.template getAnalysis(); - InstructionDeleter deleter; - auto value = arguments.takeValue(); - hoistDestroysOfOwnedLexicalValue(value, *value->getFunction(), deleter, - calleeAnalysis); - function.print(llvm::outs()); - }); -} // end namespace swift::test diff --git a/test/SILOptimizer/canonicalize_ossa_lifetime_unit.sil b/test/SILOptimizer/canonicalize_ossa_lifetime_unit.sil index 5802ab06e71da..22bf28a26fde1 100644 --- a/test/SILOptimizer/canonicalize_ossa_lifetime_unit.sil +++ b/test/SILOptimizer/canonicalize_ossa_lifetime_unit.sil @@ -56,12 +56,12 @@ sil [ossa] @retract_value_lifetime_into_access_scope_when_access_scopes_not_resp bb0(%addr : $*C): %instance = apply undef() : $@convention(thin) () -> @owned C debug_value [trace] %instance : $C - // respect access scopes - // VVVV + // respect access scopes + // VVVV specify_test "canonicalize-ossa-lifetime true false true @trace" specify_test "canonicalize-ossa-lifetime true false false @trace" - // ^^^^^ - // respect access scopes + // ^^^^^ + // respect access scopes %copy = copy_value %instance : $C %access = begin_access [modify] [static] %addr : $*C store %copy to [init] %access : $*C diff --git a/test/SILOptimizer/shrink_borrow_scope.sil b/test/SILOptimizer/shrink_borrow_scope.sil index fc689d8da959a..80f4dd8bb7afb 100644 --- a/test/SILOptimizer/shrink_borrow_scope.sil +++ b/test/SILOptimizer/shrink_borrow_scope.sil @@ -558,8 +558,8 @@ exit: // CHECK: [[LIFETIME_D:%[^,]+]] = begin_borrow [[INSTANCE_D]] // CHECK: struct $CDCase ([[LIFETIME_C]] : $C, [[LIFETIME_D]] : $D) // CHECK: end_borrow [[LIFETIME_D]] -// CHECK: destroy_value [[INSTANCE_D]] // CHECK: end_borrow [[LIFETIME_C]] +// CHECK: destroy_value [[INSTANCE_D]] // CHECK: return [[COPY_C]] // CHECK-LABEL: } // end sil function 'hoist_over_struct' sil [ossa] @hoist_over_struct : $@convention(thin) (@owned C, @owned D) -> @owned C { diff --git a/test/SILOptimizer/shrink_borrow_scope.swift b/test/SILOptimizer/shrink_borrow_scope.swift index a49042ffbe2a1..6cfb9aeb935e1 100644 --- a/test/SILOptimizer/shrink_borrow_scope.swift +++ b/test/SILOptimizer/shrink_borrow_scope.swift @@ -27,10 +27,10 @@ public func eliminate_copy_of_returned_then_consumed_owned_value(arg: __owned An // no copy of 'x' _ = consumeAndProduce(x) // CHECK: [[RESULT:%[^,]+]] = apply {{%[^,]+}}([[MOVE_X]]) - // release result // release arg - // CHECK: destroy_value [[ARG]] + // release result // CHECK: destroy_value [[RESULT]] + // CHECK: destroy_value [[ARG]] // CHECK-LABEL: } // end sil function 'eliminate_copy_of_returned_then_consumed_owned_value' } diff --git a/test/SILOptimizer/utils/pruned_liveness/are_uses_within_boundary.sil b/test/SILOptimizer/utils/pruned_liveness/are_uses_within_boundary.sil new file mode 100644 index 0000000000000..6041079d169b8 --- /dev/null +++ b/test/SILOptimizer/utils/pruned_liveness/are_uses_within_boundary.sil @@ -0,0 +1,257 @@ +// RUN: %target-sil-opt -test-runner %s -o /dev/null 2>&1 | %FileCheck %s + +import Builtin + +class Klass {} + +sil @getKlass : $@convention(thin) () -> (@owned Klass) +sil @takeKlass : $@convention(thin) (@owned Klass) -> () +sil @borrowKlass : $@convention(thin) (@guaranteed Klass) -> () + +// CHECK-LABEL: begin running test {{.*}} on dead_function_argument +// CHECK: RESULT: true +// CHECK-LABEL: end running test {{.*}} on dead_function_argument +sil [ossa] @dead_function_argument : $@convention(thin) (@owned Klass) -> () { +entry(%k : @owned $Klass): + debug_value %k : $Klass + + specify_test """ + SSAPrunedLiveness__areUsesWithinBoundary + def: %k + liveness-uses: + uses: @instruction[0].operand[0] + """ + + unreachable +} + +// CHECK-LABEL: begin running test {{.*}} on liveness_in_dead_end__before +// CHECK: RESULT: false +// CHECK-LABEL: end running test {{.*}} on liveness_in_dead_end__before +sil [ossa] @liveness_in_dead_end__before : $@convention(thin) () -> () { +entry: + %getKlass = function_ref @getKlass : $@convention(thin) () -> (@owned Klass) + %takeKlass = function_ref @takeKlass : $@convention(thin) (@owned Klass) -> () + cond_br undef, exit, die +exit: + %retval = tuple () + return %retval : $() + +die: + %val = tuple () + %inst = tuple (%val : $()) + %k = apply %getKlass() : $@convention(thin) () -> (@owned Klass) + apply %takeKlass(%k) : $@convention(thin) (@owned Klass) -> () + + specify_test """ + SSAPrunedLiveness__areUsesWithinBoundary + def: %k + liveness-uses: @block.instruction[3] ending + uses: @block.instruction[1].operand[0] + """ + + unreachable +} + +// CHECK-LABEL: begin running test {{.*}} on liveness_in_dead_end__before_2 +// CHECK: RESULT: false +// CHECK-LABEL: end running test {{.*}} on liveness_in_dead_end__before_2 +sil [ossa] @liveness_in_dead_end__before_2 : $@convention(thin) () -> () { +entry: + %getKlass = function_ref @getKlass : $@convention(thin) () -> (@owned Klass) + %takeKlass = function_ref @takeKlass : $@convention(thin) (@owned Klass) -> () + cond_br undef, exit, die +exit: + %retval = tuple () + return %retval : $() + +die: + %val = tuple () + %inst = tuple (%val : $()) + %k = apply %getKlass() : $@convention(thin) () -> (@owned Klass) + apply %takeKlass(%k) : $@convention(thin) (@owned Klass) -> () + + specify_test """ + SSAPrunedLiveness__areUsesWithinBoundary + def: %k + liveness-uses: @block.instruction[3] ending + uses: @block.instruction[1].operand[0] + """ + + br die2 +die2: + unreachable +} + +// CHECK-LABEL: begin running test {{.*}} on liveness_in_dead_end__after_consuming +// CHECK: RESULT: false +// CHECK-LABEL: end running test {{.*}} on liveness_in_dead_end__after_consuming +sil [ossa] @liveness_in_dead_end__after_consuming : $@convention(thin) () -> () { +entry: + %getKlass = function_ref @getKlass : $@convention(thin) () -> (@owned Klass) + %takeKlass = function_ref @takeKlass : $@convention(thin) (@owned Klass) -> () + cond_br undef, exit, die +exit: + %retval = tuple () + return %retval : $() + +die: + %k = apply %getKlass() : $@convention(thin) () -> (@owned Klass) + apply %takeKlass(%k) : $@convention(thin) (@owned Klass) -> () + %val = tuple () + %inst = tuple (%val : $()) + + specify_test """ + SSAPrunedLiveness__areUsesWithinBoundary + def: %k + liveness-uses: @block.instruction[1] ending + uses: @block.instruction[1].operand[0] + """ + + unreachable +} + +// CHECK-LABEL: begin running test {{.*}} on liveness_in_dead_end__after_nonconsuming +// CHECK: RESULT: true +// CHECK-LABEL: end running test {{.*}} on liveness_in_dead_end__after_nonconsuming +sil [ossa] @liveness_in_dead_end__after_nonconsuming : $@convention(thin) () -> () { +entry: + %getKlass = function_ref @getKlass : $@convention(thin) () -> (@owned Klass) + %takeKlass = function_ref @takeKlass : $@convention(thin) (@owned Klass) -> () + %borrowKlass = function_ref @borrowKlass : $@convention(thin) (@guaranteed Klass) -> () + cond_br undef, exit, die +exit: + %retval = tuple () + return %retval : $() + +die: + %k = apply %getKlass() : $@convention(thin) () -> (@owned Klass) + apply %borrowKlass(%k) : $@convention(thin) (@guaranteed Klass) -> () + %val = tuple () + %inst = tuple (%val : $()) + + specify_test """ + SSAPrunedLiveness__areUsesWithinBoundary + def: %k + liveness-uses: @block.instruction[1] non-ending + uses: @block.instruction[3].operand[0] + """ + + unreachable +} + +// CHECK-LABEL: begin running test {{.*}} on liveness_in_dead_end__after_nonuse +// CHECK: RESULT: true +// CHECK-LABEL: end running test {{.*}} on liveness_in_dead_end__after_nonuse +sil [ossa] @liveness_in_dead_end__after_nonuse : $@convention(thin) () -> () { +entry: + %getKlass = function_ref @getKlass : $@convention(thin) () -> (@owned Klass) + %takeKlass = function_ref @takeKlass : $@convention(thin) (@owned Klass) -> () + %borrowKlass = function_ref @borrowKlass : $@convention(thin) (@guaranteed Klass) -> () + cond_br undef, exit, die +exit: + %retval = tuple () + return %retval : $() + +die: + %k = apply %getKlass() : $@convention(thin) () -> (@owned Klass) + %val = tuple () + %inst = tuple (%val : $()) + + specify_test """ + SSAPrunedLiveness__areUsesWithinBoundary + def: %k + liveness-uses: @block.instruction[1] non-ending + uses: @block.instruction[2].operand[0] + """ + + unreachable +} + +// CHECK-LABEL: begin running test {{.*}} on use_in_dead_end__after_nonconsuming_liveness_boundary +// CHECK: RESULT: true +// CHECK-LABEL: end running test {{.*}} on use_in_dead_end__after_nonconsuming_liveness_boundary +sil [ossa] @use_in_dead_end__after_nonconsuming_liveness_boundary : $@convention(thin) () -> () { +entry: + %getKlass = function_ref @getKlass : $@convention(thin) () -> (@owned Klass) + %takeKlass = function_ref @takeKlass : $@convention(thin) (@owned Klass) -> () + %borrowKlass = function_ref @borrowKlass : $@convention(thin) (@guaranteed Klass) -> () + %k = apply %getKlass() : $@convention(thin) () -> (@owned Klass) + cond_br undef, exit, die +exit: + apply %takeKlass(%k) : $@convention(thin) (@owned Klass) -> () + %retval = tuple () + return %retval : $() + +die: + %val = tuple () + %inst = tuple (%val : $()) + + specify_test """ + SSAPrunedLiveness__areUsesWithinBoundary + def: %k + liveness-uses: @block[1].instruction[1] ending + uses: @block.instruction[1].operand[0] + """ + + unreachable +} + +// CHECK-LABEL: begin running test {{.*}} on use_in_dead_end__after_unavailable +// CHECK: RESULT: false +// CHECK-LABEL: end running test {{.*}} on use_in_dead_end__after_unavailable +sil [ossa] @use_in_dead_end__after_unavailable : $@convention(thin) () -> () { +entry: + %getKlass = function_ref @getKlass : $@convention(thin) () -> (@owned Klass) + %takeKlass = function_ref @takeKlass : $@convention(thin) (@owned Klass) -> () + %borrowKlass = function_ref @borrowKlass : $@convention(thin) (@guaranteed Klass) -> () + %k = apply %getKlass() : $@convention(thin) () -> (@owned Klass) + apply %takeKlass(%k) : $@convention(thin) (@owned Klass) -> () + cond_br undef, exit, die +exit: + %retval = tuple () + return %retval : $() + +die: + %val = tuple () + %inst = tuple (%val : $()) + + specify_test """ + SSAPrunedLiveness__areUsesWithinBoundary + def: %k + liveness-uses: @block[0].instruction[4] ending + uses: @block.instruction[1].operand[0] + """ + + unreachable +} + +// CHECK-LABEL: begin running test {{.*}} on use_in_dead_end__after_available_but_dead +// CHECK: RESULT: false +// CHECK-LABEL: end running test {{.*}} on use_in_dead_end__after_available_but_dead +sil [ossa] @use_in_dead_end__after_available_but_dead : $@convention(thin) () -> () { +entry: + %getKlass = function_ref @getKlass : $@convention(thin) () -> (@owned Klass) + %takeKlass = function_ref @takeKlass : $@convention(thin) (@owned Klass) -> () + %borrowKlass = function_ref @borrowKlass : $@convention(thin) (@guaranteed Klass) -> () + %k = apply %getKlass() : $@convention(thin) () -> (@owned Klass) + apply %borrowKlass(%k) : $@convention(thin) (@guaranteed Klass) -> () + cond_br undef, exit, die +exit: + apply %takeKlass(%k) : $@convention(thin) (@owned Klass) -> () + %retval = tuple () + return %retval : $() + +die: + %val = tuple () + %inst = tuple (%val : $()) + + specify_test """ + SSAPrunedLiveness__areUsesWithinBoundary + def: %k + liveness-uses: @block[0].instruction[4] non-ending + uses: @block.instruction[1].operand[0] + """ + + unreachable +}