From 2a305e17690ad837f8d3c13e24290f4a0f3cfb88 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Tue, 15 Jul 2025 11:51:45 +0100 Subject: [PATCH 01/44] Add initial structure for the new licm pass. --- .../Sources/Optimizer/Analysis/CMakeLists.txt | 1 + .../Sources/Optimizer/Analysis/LoopTree.swift | 97 ++++++ .../Optimizer/FunctionPasses/CMakeLists.txt | 1 + .../LoopInvariantCodeMotion.swift | 294 ++++++++++++++++++ .../Optimizer/PassManager/Context.swift | 5 + .../PassManager/PassRegistration.swift | 1 + .../swift/SILOptimizer/OptimizerBridging.h | 23 ++ .../SILOptimizer/OptimizerBridgingImpl.h | 38 +++ .../swift/SILOptimizer/PassManager/Passes.def | 2 + 9 files changed, 462 insertions(+) create mode 100644 SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift create mode 100644 SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt index bedf3296a61df..4f95a146351ba 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt @@ -9,6 +9,7 @@ swift_compiler_sources(Optimizer AliasAnalysis.swift CalleeAnalysis.swift + LoopTree.swift DeadEndBlocksAnalysis.swift DominatorTree.swift PostDominatorTree.swift) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift new file mode 100644 index 0000000000000..665938aa3ea45 --- /dev/null +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -0,0 +1,97 @@ +//===--- LoopTree.swift ---------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SIL +import OptimizerBridging + +/// Describes top level loops. +struct LoopTree { + let bridged: BridgedLoopTree + + let loops: TopLevelLoopArray + + init(bridged: BridgedLoopTree) { + self.bridged = bridged + self.loops = TopLevelLoopArray(bridged) + } +} + +/// Describes a loop with it's children. +struct Loop { + let bridged: BridgedLoop + + let innerLoops: LoopArray + let basicBlocks: BasicBlockArray + + init(bridged: BridgedLoop) { + self.bridged = bridged + self.innerLoops = LoopArray(bridged) + self.basicBlocks = BasicBlockArray(bridged) + } +} + +struct TopLevelLoopArray: BridgedRandomAccessCollection { + private let bridgedLoopTree: BridgedLoopTree + + public let count: Int + + public var startIndex: Int { return 0 } + public var endIndex: Int { return count } + + public init(_ bridgedLoopTree: BridgedLoopTree) { + self.bridgedLoopTree = bridgedLoopTree + self.count = bridgedLoopTree.getTopLevelLoopCount() + } + + public subscript(_ index: Int) -> Loop { + assert(index >= startIndex && index < endIndex) + return Loop(bridged: bridgedLoopTree.getLoop(index)) + } +} + +struct LoopArray: BridgedRandomAccessCollection { + private let bridgedLoop: BridgedLoop + + public let count: Int + + public var startIndex: Int { return 0 } + public var endIndex: Int { return count } + + public init(_ bridgedLoop: BridgedLoop) { + self.bridgedLoop = bridgedLoop + self.count = bridgedLoop.getInnerLoopCount() + } + + public subscript(_ index: Int) -> Loop { + assert(index >= startIndex && index < endIndex) + return Loop(bridged: bridgedLoop.getInnerLoop(index)) + } +} + +struct BasicBlockArray: BridgedRandomAccessCollection { + private let bridgedLoop: BridgedLoop + + public let count: Int + + public var startIndex: Int { return 0 } + public var endIndex: Int { return count } + + public init(_ bridgedLoop: BridgedLoop) { + self.bridgedLoop = bridgedLoop + self.count = bridgedLoop.getBasicBlockCount() + } + + public subscript(_ index: Int) -> BasicBlock { + assert(index >= startIndex && index < endIndex) + return bridgedLoop.getBasicBlock(index).block + } +} diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt index 52fe55235aee5..c0d7d0fada55e 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt @@ -25,6 +25,7 @@ swift_compiler_sources(Optimizer LifetimeDependenceDiagnostics.swift LifetimeDependenceInsertion.swift LifetimeDependenceScopeFixup.swift + LoopInvariantCodeMotion.swift ObjectOutliner.swift ObjCBridgingOptimization.swift MergeCondFails.swift diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift new file mode 100644 index 0000000000000..e66ce13a8164f --- /dev/null +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -0,0 +1,294 @@ +//===--- LoopInvariantCodeMotion.swift ------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SIL + +struct SupportingDataStructures { + let runsOnHighLevelSil: Bool + let domTree: DominatorTree + let postDomTree: PostDominatorTree + let aliasAnalysis: AliasAnalysis + let calleeAnalysis: CalleeAnalysis +} + +struct DiscoveredMovableInstructions { + let context: Context + + var loadsAndStores: InstructionSet + var hoistUp: InstructionSet + var sinkDown: InstructionSet + var specialHoist: InstructionSet + + init(context: Context) { + self.context = context + + self.loadsAndStores = InstructionSet(context) + self.hoistUp = InstructionSet(context) + self.sinkDown = InstructionSet(context) + self.specialHoist = InstructionSet(context) + } + + mutating func clear() { + deinitialize() + + self.loadsAndStores = InstructionSet(context) + self.hoistUp = InstructionSet(context) + self.sinkDown = InstructionSet(context) + self.specialHoist = InstructionSet(context) + } + + mutating func deinitialize() { + loadsAndStores.deinitialize() + hoistUp.deinitialize() + sinkDown.deinitialize() + specialHoist.deinitialize() + } +} + +let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion") { function, context in + let loopTree = context.loopTree + + guard !loopTree.loops.isEmpty else { return } + + let dataStructures = SupportingDataStructures( + runsOnHighLevelSil: true, // TODO: Make a parameter. + domTree: context.dominatorTree, + postDomTree: context.postDominatorTree, + aliasAnalysis: context.aliasAnalysis, + calleeAnalysis: context.calleeAnalysis + ) + + var changed = false + + for loop in loopTree.loops { + changed = optimizeTopLevelLoop( + topLevelLoop: loop, + dataStructures: dataStructures, + context: context + ) || changed + } +} + +private func optimizeTopLevelLoop( + topLevelLoop: Loop, + dataStructures: SupportingDataStructures, + context: Context +) -> Bool { + var workList = [topLevelLoop] + var i = 0 + while i < workList.count { + let thisLoop = workList[i] + workList.append(contentsOf: thisLoop.innerLoops) + i += 1 + } + + var movableInstructions = DiscoveredMovableInstructions(context: context) + + var changed = false + + while !workList.isEmpty { + let thisLoop = workList.popLast()! + + // TODO: Compute and propagate the summaries (Might not be necessary) + + var thisLoopChanged = false + + repeat { + analyzeLoop(loop: thisLoop, dataStructures: dataStructures, movableInstructions: &movableInstructions, context: context) + + thisLoopChanged = optimizeLoop(loop: thisLoop, movableInstructions: movableInstructions) + + if thisLoopChanged { + changed = true + } + + movableInstructions.clear() + } while thisLoopChanged + } + + movableInstructions.deinitialize() + + return changed +} + +private func analyzeLoop( + loop: Loop, + dataStructures: SupportingDataStructures, + movableInstructions: inout DiscoveredMovableInstructions, + context: Context +) { + let preheader = loop.bridged.getPreheader() + + var readOnlyApplies = Stack(context) + var globalInitCalls = Stack(context) + + var loads = Stack(context) + var stores = Stack(context) + var fixLifetimes = Stack(context) + var beginAccesses = Stack(context) + var fullApplies = Stack(context) + + var hasOtherMemReadingInsts = false + + for bb in loop.basicBlocks { + var blockSideEffects = InstructionSet(context) + defer { + blockSideEffects.deinitialize() + } + + for inst in bb.instructions { + if hasOwnershipOperandsOrResults(inst: inst) { + checkSideEffects( + inst: inst, + blockSideEffects: &blockSideEffects, + hasOtherMemReadingInsts: &hasOtherMemReadingInsts + ) + + guard let fullApply = inst as? FullApplySite else { continue } + + fullApplies.push(fullApply) + } + + switch inst { + case let fixLifetimeInst as FixLifetimeInst: + if dataStructures.domTree.bridged.dominates(fixLifetimeInst.operand.value.parentBlock.bridged, preheader) { + fixLifetimes.push(fixLifetimeInst) + } + case let loadInst as LoadInst: + loads.push(loadInst) + movableInstructions.loadsAndStores.insert(loadInst) + case let storeInst as StoreInst: + switch storeInst.storeOwnership { + case .assign, .initialize: + continue // TODO: Add support + case .unqualified, .trivial: + break + } + stores.push(storeInst) + movableInstructions.loadsAndStores.insert(storeInst) + checkSideEffects( + inst: storeInst, + blockSideEffects: &blockSideEffects, + hasOtherMemReadingInsts: &hasOtherMemReadingInsts + ) + case let beginAccessInst as BeginAccessInst: + beginAccesses.push(beginAccessInst) + checkSideEffects( + inst: beginAccessInst, + blockSideEffects: &blockSideEffects, + hasOtherMemReadingInsts: &hasOtherMemReadingInsts + ) + case let refElementAddrInst as RefElementAddrInst: + movableInstructions.specialHoist.insert(refElementAddrInst) + case let condFailInst as CondFailInst: + movableInstructions.hoistUp.insert(condFailInst) + checkSideEffects( + inst: condFailInst, + blockSideEffects: &blockSideEffects, + hasOtherMemReadingInsts: &hasOtherMemReadingInsts + ) + case let applyInst as ApplyInst: + // TODO: Handle apply inst + fallthrough + default: + switch inst { + case let fullApply as FullApplySite: + fullApplies.push(fullApply) + case let builtinInst as BuiltinInst: + switch builtinInst.id { + case .Once, .OnceWithContext: + // TODO: Check if it collides with global init + default: break; + } + default: break + } + + checkSideEffects(inst: inst, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts) + + if canHoistUpDefault(inst: inst, loop: loop, dataStructures: dataStructures) { + movableInstructions.hoistUp.insert(inst) + } + } + } + } + + // TODO: Iterated through the stacks with extracted instructions. +} + +private func optimizeLoop( + loop: Loop, + movableInstructions: DiscoveredMovableInstructions +) -> Bool { + // TODO: Optimize this loop + return false +} + +private func checkSideEffects( + inst: Instruction, + blockSideEffects: inout InstructionSet, + hasOtherMemReadingInsts: inout Bool +) { + if inst.mayHaveSideEffects { + // TODO: Might have to propagate side effects to function wide side effects set. + blockSideEffects.insert(inst) + } else if inst.mayReadFromMemory { + hasOtherMemReadingInsts = true + } +} + +private func hasOwnershipOperandsOrResults(inst: Instruction) -> Bool { + guard inst.parentFunction.hasOwnership else { return false } + + // TODO: Double check whether .nonUse is equivalent to .none + if inst.results.contains(where: { $0.ownership != .none }) || + inst.operands.contains(where: { $0.ownership != .nonUse }) { + return true + } + + return false +} + +private func isSafeReadOnlyApply( + applyInst: ApplyInst, + dataStructures: SupportingDataStructures +) -> Bool { + // TODO: Bridge and check applyInst.getSingleResult() + + if dataStructures.runsOnHighLevelSil { + // TODO: Figure out how to bridge and check array semantics call. + } + + // TODO: Bridge calleeAnalysis.getMemoryBehavior() + return false +} + +private func canHoistUpDefault( + inst: Instruction, + loop: Loop, + dataStructures: SupportingDataStructures +) -> Bool { + let preheader = loop.bridged.getPreheader() + // TODO: Preheader could be null_ptr. Check how to bridge optional. + + if inst is TermInst || inst is Allocation || inst is Deallocation { + return false + } + + // TODO: Handle properly the array semantics call. + + // TODO: Is this equivalent to inst->getMemoryBehavior() == MemoryBehavior::None + if inst.memoryEffects == .noEffects { + return true + } + + return false +} diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift index 393960e6329f5..0891f83830435 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift @@ -326,6 +326,11 @@ struct FunctionPassContext : MutatingContext { let bridgedPDT = _bridged.getPostDomTree() return PostDominatorTree(bridged: bridgedPDT) } + + var loopTree: LoopTree { + let bridgedLT = _bridged.getLoopTree() + return LoopTree(bridged: bridgedLT) + } var swiftArrayDecl: NominalTypeDecl { _bridged.getSwiftArrayDecl().getAs(NominalTypeDecl.self) diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift index 16cfe109e6bf7..ee48cc4ca5f9a 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift @@ -107,6 +107,7 @@ private func registerSwiftPasses() { registerPass(tempLValueElimination, { tempLValueElimination.run($0) }) registerPass(generalClosureSpecialization, { generalClosureSpecialization.run($0) }) registerPass(autodiffClosureSpecialization, { autodiffClosureSpecialization.run($0) }) + registerPass(loopInvariantCodeMotionPass, { loopInvariantCodeMotionPass.run($0) }) // Instruction passes registerForSILCombine(BeginBorrowInst.self, { run(BeginBorrowInst.self, $0) }) diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index e70f44cfb1506..ec0b761f079ae 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -45,6 +45,8 @@ class CalleeList; class DeadEndBlocks; class DominanceInfo; class PostDominanceInfo; +class SILLoopInfo; +class SILLoop; class BasicBlockSet; class NodeSet; class OperandSet; @@ -128,6 +130,25 @@ struct BridgedPostDomTree { BRIDGED_INLINE bool postDominates(BridgedBasicBlock dominating, BridgedBasicBlock dominated) const; }; +struct BridgedLoop { + swift::SILLoop * _Nonnull l; + + BRIDGED_INLINE SwiftInt getInnerLoopCount() const; + BRIDGED_INLINE BridgedLoop getInnerLoop(SwiftInt index) const; + + BRIDGED_INLINE SwiftInt getBasicBlockCount() const; + BRIDGED_INLINE BridgedBasicBlock getBasicBlock(SwiftInt index) const; + + BRIDGED_INLINE BridgedBasicBlock getPreheader() const; +}; + +struct BridgedLoopTree { + swift::SILLoopInfo * _Nonnull li; + + BRIDGED_INLINE SwiftInt getTopLevelLoopCount() const; + BRIDGED_INLINE BridgedLoop getLoop(SwiftInt index) const; +}; + struct BridgedUtilities { typedef void (* _Nonnull VerifyFunctionFn)(BridgedPassContext, BridgedFunction); typedef void (* _Nonnull UpdateFunctionFn)(BridgedPassContext, BridgedFunction); @@ -219,6 +240,8 @@ struct BridgedPassContext { SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedPostDomTree getPostDomTree() const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedDeclObj getSwiftArrayDecl() const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedDeclObj getSwiftMutableSpanDecl() const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedLoopTree getLoopTree() const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedLoop getLoop() const; // AST diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index 0db955f751770..418b910542511 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -24,6 +24,7 @@ #include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h" #include "swift/SILOptimizer/Analysis/DeadEndBlocksAnalysis.h" #include "swift/SILOptimizer/Analysis/DominanceAnalysis.h" +#include "swift/SILOptimizer/Analysis/LoopAnalysis.h" #include "swift/SILOptimizer/OptimizerBridging.h" #include "swift/SILOptimizer/PassManager/PassManager.h" #include "swift/SILOptimizer/PassManager/Transforms.h" @@ -87,6 +88,38 @@ bool BridgedPostDomTree::postDominates(BridgedBasicBlock dominating, BridgedBasi return pdi->dominates(dominating.unbridged(), dominated.unbridged()); } +//===----------------------------------------------------------------------===// +// BridgedLoopTree, BridgedLoop +//===----------------------------------------------------------------------===// + +SwiftInt BridgedLoopTree::getTopLevelLoopCount() const { + return li->end() - li->begin(); +} + +BridgedLoop BridgedLoopTree::getLoop(SwiftInt index) const { + return {li->begin()[index]}; +} + +SwiftInt BridgedLoop::getInnerLoopCount() const { + return l->end() - l->begin(); +} + +BridgedLoop BridgedLoop::getInnerLoop(SwiftInt index) const { + return {l->begin()[index]}; +} + +SwiftInt BridgedLoop::getBasicBlockCount() const { + return l->getBlocks().size(); +} + +BridgedBasicBlock BridgedLoop::getBasicBlock(SwiftInt index) const { + return {l->getBlocks()[index]}; +} + +BridgedBasicBlock BridgedLoop::getPreheader() const { + return {l->getLoopPreheader()}; +} + //===----------------------------------------------------------------------===// // BridgedBasicBlockSet //===----------------------------------------------------------------------===// @@ -216,6 +249,11 @@ BridgedPostDomTree BridgedPassContext::getPostDomTree() const { return {pda->get(invocation->getFunction())}; } +BridgedLoopTree BridgedPassContext::getLoopTree() const { + auto *lt = invocation->getPassManager()->getAnalysis(); + return {lt->get(invocation->getFunction())}; +} + BridgedDeclObj BridgedPassContext::getSwiftArrayDecl() const { swift::SILModule *mod = invocation->getPassManager()->getModule(); return {mod->getASTContext().getArrayDecl()}; diff --git a/include/swift/SILOptimizer/PassManager/Passes.def b/include/swift/SILOptimizer/PassManager/Passes.def index a87327b70711a..a68a0ac379c53 100644 --- a/include/swift/SILOptimizer/PassManager/Passes.def +++ b/include/swift/SILOptimizer/PassManager/Passes.def @@ -148,6 +148,8 @@ PASS(TempRValueElimination, "temp-rvalue-elimination", "Remove short-lived immutable temporary copies") PASS(TempLValueElimination, "temp-lvalue-elimination", "Remove short-lived immutable temporary l-values") +PASS(LoopInvariantCodeMotion, "loop-invariant-code-motion", + "New Loop Invariant Code Motion") // NOTE - ExperimentalSwiftBasedClosureSpecialization and AutodiffClosureSpecialization are a WIP PASS(ExperimentalSwiftBasedClosureSpecialization, "experimental-swift-based-closure-specialization", From e2ada8565272eaa3c2135da4b133d6aa9fb2ae44 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Tue, 15 Jul 2025 19:06:01 +0100 Subject: [PATCH 02/44] Complete instruction extraction in analyzeLoop. Add more filtering after extraction. --- .../Sources/Optimizer/Analysis/LoopTree.swift | 34 +- .../LoopInvariantCodeMotion.swift | 373 +++++++++++++----- .../Sources/SIL/Effects.swift | 12 + .../SIL/Utilities/SmallProjectionPath.swift | 12 + .../swift/SILOptimizer/OptimizerBridging.h | 2 +- .../SILOptimizer/OptimizerBridgingImpl.h | 2 +- 6 files changed, 325 insertions(+), 110 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index 665938aa3ea45..1cd27c0890b2c 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -15,31 +15,42 @@ import OptimizerBridging /// Describes top level loops. struct LoopTree { - let bridged: BridgedLoopTree + private let context: Context + private let bridged: BridgedLoopTree let loops: TopLevelLoopArray - init(bridged: BridgedLoopTree) { + init(bridged: BridgedLoopTree, context: Context) { + self.context = context self.bridged = bridged - self.loops = TopLevelLoopArray(bridged) + self.loops = TopLevelLoopArray(bridged, context: context) } } /// Describes a loop with it's children. struct Loop { - let bridged: BridgedLoop + private let context: Context + private let bridged: BridgedLoop let innerLoops: LoopArray let basicBlocks: BasicBlockArray + let basicBlockSet: BasicBlockSet - init(bridged: BridgedLoop) { + var preheader: BasicBlock? { + bridged.getPreheader().block + } + + init(bridged: BridgedLoop, context: Context) { + self.context = context self.bridged = bridged - self.innerLoops = LoopArray(bridged) + self.innerLoops = LoopArray(bridged, context: context) self.basicBlocks = BasicBlockArray(bridged) + self.basicBlockSet = BasicBlockSet(insertContentsOf: basicBlocks, context) } } struct TopLevelLoopArray: BridgedRandomAccessCollection { + private let context: Context private let bridgedLoopTree: BridgedLoopTree public let count: Int @@ -47,18 +58,20 @@ struct TopLevelLoopArray: BridgedRandomAccessCollection { public var startIndex: Int { return 0 } public var endIndex: Int { return count } - public init(_ bridgedLoopTree: BridgedLoopTree) { + public init(_ bridgedLoopTree: BridgedLoopTree, context: Context) { + self.context = context self.bridgedLoopTree = bridgedLoopTree self.count = bridgedLoopTree.getTopLevelLoopCount() } public subscript(_ index: Int) -> Loop { assert(index >= startIndex && index < endIndex) - return Loop(bridged: bridgedLoopTree.getLoop(index)) + return Loop(bridged: bridgedLoopTree.getLoop(index), context: context) } } struct LoopArray: BridgedRandomAccessCollection { + private let context: Context private let bridgedLoop: BridgedLoop public let count: Int @@ -66,14 +79,15 @@ struct LoopArray: BridgedRandomAccessCollection { public var startIndex: Int { return 0 } public var endIndex: Int { return count } - public init(_ bridgedLoop: BridgedLoop) { + public init(_ bridgedLoop: BridgedLoop, context: Context) { + self.context = context self.bridgedLoop = bridgedLoop self.count = bridgedLoop.getInnerLoopCount() } public subscript(_ index: Int) -> Loop { assert(index >= startIndex && index < endIndex) - return Loop(bridged: bridgedLoop.getInnerLoop(index)) + return Loop(bridged: bridgedLoop.getInnerLoop(index), context: context) } } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index e66ce13a8164f..738a7e049f20f 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -12,34 +12,23 @@ import SIL -struct SupportingDataStructures { - let runsOnHighLevelSil: Bool - let domTree: DominatorTree - let postDomTree: PostDominatorTree - let aliasAnalysis: AliasAnalysis - let calleeAnalysis: CalleeAnalysis +let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion") { function, context in + for loop in context.loopTree.loops { + optimizeTopLevelLoop( + topLevelLoop: loop, + runsOnHighLevelSil: true, // TODO: Make a parameter. + context: context + ) + } } struct DiscoveredMovableInstructions { - let context: Context - var loadsAndStores: InstructionSet var hoistUp: InstructionSet var sinkDown: InstructionSet var specialHoist: InstructionSet init(context: Context) { - self.context = context - - self.loadsAndStores = InstructionSet(context) - self.hoistUp = InstructionSet(context) - self.sinkDown = InstructionSet(context) - self.specialHoist = InstructionSet(context) - } - - mutating func clear() { - deinitialize() - self.loadsAndStores = InstructionSet(context) self.hoistUp = InstructionSet(context) self.sinkDown = InstructionSet(context) @@ -54,35 +43,12 @@ struct DiscoveredMovableInstructions { } } -let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion") { function, context in - let loopTree = context.loopTree - - guard !loopTree.loops.isEmpty else { return } - - let dataStructures = SupportingDataStructures( - runsOnHighLevelSil: true, // TODO: Make a parameter. - domTree: context.dominatorTree, - postDomTree: context.postDominatorTree, - aliasAnalysis: context.aliasAnalysis, - calleeAnalysis: context.calleeAnalysis - ) - - var changed = false - - for loop in loopTree.loops { - changed = optimizeTopLevelLoop( - topLevelLoop: loop, - dataStructures: dataStructures, - context: context - ) || changed - } -} - private func optimizeTopLevelLoop( topLevelLoop: Loop, - dataStructures: SupportingDataStructures, - context: Context -) -> Bool { + runsOnHighLevelSil: Bool, + context: FunctionPassContext, +) { + // FIXME: Make a recursive function that populates the stack work list. Efficiency purposes. var workList = [topLevelLoop] var i = 0 while i < workList.count { @@ -91,56 +57,61 @@ private func optimizeTopLevelLoop( i += 1 } - var movableInstructions = DiscoveredMovableInstructions(context: context) - - var changed = false - - while !workList.isEmpty { - let thisLoop = workList.popLast()! - - // TODO: Compute and propagate the summaries (Might not be necessary) - + while let thisLoop = workList.popLast() { var thisLoopChanged = false repeat { - analyzeLoop(loop: thisLoop, dataStructures: dataStructures, movableInstructions: &movableInstructions, context: context) - - thisLoopChanged = optimizeLoop(loop: thisLoop, movableInstructions: movableInstructions) - - if thisLoopChanged { - changed = true + guard let movableInstructions = analyzeLoop( + loop: thisLoop, + runsOnHighLevelSil: runsOnHighLevelSil, + context: context + ) else { + return // Encountered a loop without preheader. Return early. } - movableInstructions.clear() + thisLoopChanged = optimizeLoop(loop: thisLoop, movableInstructions: movableInstructions) } while thisLoopChanged } - - movableInstructions.deinitialize() - - return changed } private func analyzeLoop( loop: Loop, - dataStructures: SupportingDataStructures, - movableInstructions: inout DiscoveredMovableInstructions, - context: Context -) { - let preheader = loop.bridged.getPreheader() + runsOnHighLevelSil: Bool, + context: FunctionPassContext +) -> DiscoveredMovableInstructions? { + guard let preheader = loop.preheader else { + return nil + } + + var movableInstructions = DiscoveredMovableInstructions(context: context) + defer { + movableInstructions.deinitialize() + } var readOnlyApplies = Stack(context) var globalInitCalls = Stack(context) + var loopSideEffects = Stack(context) var loads = Stack(context) var stores = Stack(context) var fixLifetimes = Stack(context) var beginAccesses = Stack(context) var fullApplies = Stack(context) + defer { + readOnlyApplies.deinitialize() + globalInitCalls.deinitialize() + loopSideEffects.deinitialize() + loads.deinitialize() + stores.deinitialize() + fixLifetimes.deinitialize() + beginAccesses.deinitialize() + fullApplies.deinitialize() + } var hasOtherMemReadingInsts = false for bb in loop.basicBlocks { - var blockSideEffects = InstructionSet(context) + var blockSideEffects = Stack(context) defer { blockSideEffects.deinitialize() } @@ -149,6 +120,7 @@ private func analyzeLoop( if hasOwnershipOperandsOrResults(inst: inst) { checkSideEffects( inst: inst, + loopSideEffects: &loopSideEffects, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts ) @@ -160,7 +132,7 @@ private func analyzeLoop( switch inst { case let fixLifetimeInst as FixLifetimeInst: - if dataStructures.domTree.bridged.dominates(fixLifetimeInst.operand.value.parentBlock.bridged, preheader) { + if fixLifetimeInst.parentBlock.dominates(preheader, context.dominatorTree) { fixLifetimes.push(fixLifetimeInst) } case let loadInst as LoadInst: @@ -177,6 +149,7 @@ private func analyzeLoop( movableInstructions.loadsAndStores.insert(storeInst) checkSideEffects( inst: storeInst, + loopSideEffects: &loopSideEffects, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts ) @@ -184,6 +157,7 @@ private func analyzeLoop( beginAccesses.push(beginAccessInst) checkSideEffects( inst: beginAccessInst, + loopSideEffects: &loopSideEffects, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts ) @@ -193,11 +167,23 @@ private func analyzeLoop( movableInstructions.hoistUp.insert(condFailInst) checkSideEffects( inst: condFailInst, + loopSideEffects: &loopSideEffects, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts ) case let applyInst as ApplyInst: - // TODO: Handle apply inst + if isSafeReadOnlyApply( + applyInst: applyInst, + runsOnHighLevelSil: runsOnHighLevelSil, + calleeAnalysis: context.calleeAnalysis + ) { + readOnlyApplies.push(applyInst) + } else if let callee = applyInst.referencedFunction, + callee.isGlobalInitFunction, + !mayConflictWithGlobalInit(globalInitCall: applyInst, sideEffects: blockSideEffects, aliasAnalysis: context.aliasAnalysis){ + globalInitCalls.push(applyInst) + } + fallthrough default: switch inst { @@ -206,22 +192,80 @@ private func analyzeLoop( case let builtinInst as BuiltinInst: switch builtinInst.id { case .Once, .OnceWithContext: - // TODO: Check if it collides with global init + if !mayConflictWithGlobalInit(globalInitCall: builtinInst, sideEffects: blockSideEffects, aliasAnalysis: context.aliasAnalysis) { + globalInitCalls.push(builtinInst) + } default: break; } default: break } - checkSideEffects(inst: inst, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts) + checkSideEffects( + inst: inst, + loopSideEffects: &loopSideEffects, + blockSideEffects: &blockSideEffects, + hasOtherMemReadingInsts: &hasOtherMemReadingInsts + ) - if canHoistUpDefault(inst: inst, loop: loop, dataStructures: dataStructures) { + if canHoistUpDefault(inst: inst, loop: loop, domTree: context.dominatorTree) { movableInstructions.hoistUp.insert(inst) } } } } - // TODO: Iterated through the stacks with extracted instructions. + // TODO: Check for size to avoid quadratic complexity + for readOnlyApply in readOnlyApplies { + if !mayWriteTo( + readOnlyApply, + sideEffects: loopSideEffects, + aliasAnalysis: context.aliasAnalysis, + calleeAnalysis: context.calleeAnalysis + ) { + movableInstructions.hoistUp.insert(readOnlyApply) + } + } + + // TODO: Check for size to avoid quadratic complexity + for load in loads { + if !mayWriteTo(load, sideEffects: loopSideEffects, aliasAnalysis: context.aliasAnalysis) { + movableInstructions.hoistUp.insert(load) + } + } + + if !globalInitCalls.isEmpty { + // TODO: Check for post dom tree root node. Check if post dominated side effects conflict with global initializer call. + } + + if !hasOtherMemReadingInsts { + for storeInst in stores { + let accessPath = storeInst.destination.accessPath + + if accessPath.isLoopInvariant(loop: loop) { + // TODO: Add remaining checks. + } + } + } + + if !fixLifetimes.isEmpty { + let sideEffectsMayRelease = loopSideEffects.contains(where: { $0.mayRelease }) + + for fixLifetime in fixLifetimes { + guard fixLifetime.operand.value.type.isAddress else { continue } + + if sideEffectsMayRelease || !mayWriteTo(fixLifetime, sideEffects: loopSideEffects, aliasAnalysis: context.aliasAnalysis) { + movableInstructions.sinkDown.insert(fixLifetime) + } + } + } + + for beginAccess in beginAccesses { + guard handledEndAccess(beginAccessInst: beginAccess, loop: loop, context: context) else { continue } + + // TODO: Analyze begin access and add instructions to the special hoist. + } + + return movableInstructions } private func optimizeLoop( @@ -234,12 +278,13 @@ private func optimizeLoop( private func checkSideEffects( inst: Instruction, - blockSideEffects: inout InstructionSet, + loopSideEffects: inout Stack, + blockSideEffects: inout Stack, hasOtherMemReadingInsts: inout Bool ) { if inst.mayHaveSideEffects { - // TODO: Might have to propagate side effects to function wide side effects set. - blockSideEffects.insert(inst) + loopSideEffects.push(inst) + blockSideEffects.push(inst) } else if inst.mayReadFromMemory { hasOtherMemReadingInsts = true } @@ -248,47 +293,179 @@ private func checkSideEffects( private func hasOwnershipOperandsOrResults(inst: Instruction) -> Bool { guard inst.parentFunction.hasOwnership else { return false } - // TODO: Double check whether .nonUse is equivalent to .none - if inst.results.contains(where: { $0.ownership != .none }) || - inst.operands.contains(where: { $0.ownership != .nonUse }) { - return true - } - - return false + return inst.results.contains(where: { $0.ownership != .none }) || + inst.operands.contains(where: { $0.value.ownership != .none }) } private func isSafeReadOnlyApply( applyInst: ApplyInst, - dataStructures: SupportingDataStructures + runsOnHighLevelSil: Bool, + calleeAnalysis: CalleeAnalysis ) -> Bool { - // TODO: Bridge and check applyInst.getSingleResult() + guard applyInst.functionConvention.results.allSatisfy({ $0.convention == .unowned }) else { + return false + } - if dataStructures.runsOnHighLevelSil { - // TODO: Figure out how to bridge and check array semantics call. + if runsOnHighLevelSil, + let callee = applyInst.referencedFunction, + callee.hasSemanticsAttribute("array.props.isNativeTypeChecked") { + return false } - // TODO: Bridge calleeAnalysis.getMemoryBehavior() - return false + return calleeAnalysis.getSideEffects(ofApply: applyInst).isOnlyReading } private func canHoistUpDefault( inst: Instruction, loop: Loop, - dataStructures: SupportingDataStructures + domTree: DominatorTree ) -> Bool { - let preheader = loop.bridged.getPreheader() - // TODO: Preheader could be null_ptr. Check how to bridge optional. + guard let preheader = loop.preheader else { + return false + } if inst is TermInst || inst is Allocation || inst is Deallocation { return false } // TODO: Handle properly the array semantics call. + // FIXME: Hoist the entire cpp thing to avoid reimplementing a lot of stuff. - // TODO: Is this equivalent to inst->getMemoryBehavior() == MemoryBehavior::None if inst.memoryEffects == .noEffects { return true } return false } + +private func mayWriteTo( + _ applyInst: ApplyInst, + sideEffects: Stack, + aliasAnalysis: AliasAnalysis, + calleeAnalysis: CalleeAnalysis +) -> Bool { + guard calleeAnalysis.getSideEffects(ofApply: applyInst).memory == .noEffects else { + return false + } + + for sideEffect in sideEffects { + switch sideEffect { + case let storeInst as StoreInst: + if storeInst.storeOwnership == .assign && + applyInst.mayRead(fromAddress: storeInst.destination, aliasAnalysis) { + return true; + } + case let copyAddrInst as CopyAddrInst: + if !copyAddrInst.isInitializationOfDestination && + applyInst.mayRead(fromAddress: copyAddrInst.destination, aliasAnalysis) { + return true + } + case is ApplyInst, is BeginApplyInst, is TryApplyInst: + if !calleeAnalysis.getSideEffects(ofApply: applyInst).isOnlyReading { + return true + } + case is CondFailInst, is StrongRetainInst, is UnmanagedRetainValueInst, + is RetainValueInst, is StrongRetainUnownedInst, is FixLifetimeInst, + is KeyPathInst, is DeallocStackInst, is DeallocStackRefInst, + is DeallocRefInst: + break + default: + if sideEffect.mayWriteToMemory { + return true + } + } + } + + return false +} + +private func mayWriteTo( + _ unaryInst: UnaryInstruction, + sideEffects: Stack, + aliasAnalysis: AliasAnalysis +) -> Bool { + return sideEffects + .contains { sideEffect in + sideEffect.mayWrite(toAddress: unaryInst.operand.value, aliasAnalysis) + } +} + +private func handledEndAccess(beginAccessInst: BeginAccessInst, loop: Loop, context: Context) -> Bool { + let endAccesses = getEndAccesses(beginAccessInst: beginAccessInst, context: context) + + return !endAccesses.isEmpty && !endAccesses + .contains { user in + !loop.basicBlockSet.contains(user.parentBlock) + } +} + +private func getEndAccesses(beginAccessInst: BeginAccessInst, context: Context) -> Stack { + var endAccesses = Stack(context) + defer { + endAccesses.deinitialize() + } + + endAccesses.append(contentsOf: beginAccessInst.uses.compactMap { user in + user.instruction as? EndAccessInst + }) + + return endAccesses +} + +private func mayConflictWithGlobalInit( + globalInitCall: Instruction, + sideEffect: Instruction, + aliasAnalysis: AliasAnalysis +) -> Bool { + switch sideEffect { + case let storeInst as StoreInst: + return globalInitCall.mayReadOrWrite(address: storeInst.destinationOperand.value, aliasAnalysis) + case let loadInst as LoadInst: + return globalInitCall.mayWrite(toAddress: loadInst.operand.value, aliasAnalysis) + case is CondFailInst: + return false + default: + return true + } +} + +private func mayConflictWithGlobalInit( + globalInitCall: Instruction, + sideEffects: Stack, + aliasAnalysis: AliasAnalysis +) -> Bool { + return sideEffects + .contains { sideEffect in + mayConflictWithGlobalInit( + globalInitCall: globalInitCall, + sideEffect: sideEffect, + aliasAnalysis: aliasAnalysis + ) + } +} + +private extension AccessPath { + func isLoopInvariant(loop: Loop) -> Bool { + switch base { + case .argument(let functionArgument): + // TODO: Isn't function argument guaranteed to be an invariant? + if loop.basicBlockSet.contains(functionArgument.parentBlock) { + return false + } + case .box(let inst as Instruction), .class(let inst as Instruction), .index(let inst as Instruction), + .pointer(let inst as Instruction), .stack(let inst as Instruction), .storeBorrow(let inst as Instruction), + .tail(let inst as Instruction): + if loop.basicBlockSet.contains(inst.parentBlock) { + return false + } + case .global: + break + case .yield: + break // TODO: What is yield? + case .unidentified: + return false + } + + return !projectionPath.isConstant // TODO: Is this correct? + } +} diff --git a/SwiftCompilerSources/Sources/SIL/Effects.swift b/SwiftCompilerSources/Sources/SIL/Effects.swift index 5cd2ea66c19a3..311f4423c60aa 100644 --- a/SwiftCompilerSources/Sources/SIL/Effects.swift +++ b/SwiftCompilerSources/Sources/SIL/Effects.swift @@ -505,6 +505,14 @@ public struct SideEffects : CustomStringConvertible, NoReflectionChildren { /// This is true when the function (or a callee, transitively) contains a /// deinit barrier instruction. public var isDeinitBarrier: Bool + + public static var noEffects: GlobalEffects { + return GlobalEffects(memory: .noEffects, ownership: .noEffects, allocates: false, isDeinitBarrier: false) + } + + public var isOnlyReading: Bool { + return !memory.write && ownership == .noEffects && !allocates && !isDeinitBarrier + } /// When called with default arguments, it creates an "effect-free" GlobalEffects. public init(memory: Memory = Memory(read: false, write: false), @@ -643,6 +651,10 @@ public struct SideEffects : CustomStringConvertible, NoReflectionChildren { copy = copy || other.copy destroy = destroy || other.destroy } + + public static var noEffects: Ownership { + return Ownership(copy: false, destroy: false) + } public static var worstEffects: Ownership { Ownership(copy: true, destroy: true) diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift b/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift index ea94f2771e83f..764d319aac78e 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift @@ -534,6 +534,18 @@ public struct SmallProjectionPath : Hashable, CustomStringConvertible, NoReflect return false } } + + public var isConstant: Bool { + let (kind, _, subPath) = pop() + switch kind { + case .root: + return true + case .structField, .tupleField, .enumCase, .classField, .existential: + return subPath.isConstant + default: + return false + } + } } //===----------------------------------------------------------------------===// diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index ec0b761f079ae..80075ec9b27c9 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -139,7 +139,7 @@ struct BridgedLoop { BRIDGED_INLINE SwiftInt getBasicBlockCount() const; BRIDGED_INLINE BridgedBasicBlock getBasicBlock(SwiftInt index) const; - BRIDGED_INLINE BridgedBasicBlock getPreheader() const; + BRIDGED_INLINE OptionalBridgedBasicBlock getPreheader() const; }; struct BridgedLoopTree { diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index 418b910542511..cf2d5308b740c 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -116,7 +116,7 @@ BridgedBasicBlock BridgedLoop::getBasicBlock(SwiftInt index) const { return {l->getBlocks()[index]}; } -BridgedBasicBlock BridgedLoop::getPreheader() const { +OptionalBridgedBasicBlock BridgedLoop::getPreheader() const { return {l->getLoopPreheader()}; } From 23a00266710cab3f170ec46ca630c202a00471df Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Thu, 17 Jul 2025 12:39:50 +0100 Subject: [PATCH 03/44] Implement the remainder of besides . --- .../Analysis/ArraySemanticsCall.swift | 24 + .../LoopInvariantCodeMotion.swift | 434 +++++++++++++++--- .../Optimizer/PassManager/Context.swift | 2 +- .../swift/SILOptimizer/OptimizerBridging.h | 29 ++ .../SILOptimizer/OptimizerBridgingImpl.h | 15 + 5 files changed, 438 insertions(+), 66 deletions(-) create mode 100644 SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift new file mode 100644 index 0000000000000..515bf2cf11d33 --- /dev/null +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift @@ -0,0 +1,24 @@ +//===--- ArraySemanticsCall.swift -----------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SIL +import OptimizerBridging + +struct ArraySemanticsCall { + public static func getArraySemanticsCallKind(inst: Instruction) -> BridgedArrayCallKind { + return BridgedArraySemanticsCall.getArraySemanticsCallKind(inst.bridged) + } + + public static func canHoist(inst: Instruction, to toInst: Instruction, domTree: DominatorTree) -> Bool { + return BridgedArraySemanticsCall.canHoist(inst.bridged, toInst.bridged, domTree.bridged) + } +} diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 738a7e049f20f..1e0336bd588dc 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -23,23 +23,35 @@ let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion } struct DiscoveredMovableInstructions { - var loadsAndStores: InstructionSet - var hoistUp: InstructionSet - var sinkDown: InstructionSet - var specialHoist: InstructionSet + var toDelete: InstructionSet + + var loadsAndStores: Stack + var hoistUp: Stack + var sinkDown: Stack + var specialHoist: Stack + + var loadAndStoreAddrs: Stack init(context: Context) { - self.loadsAndStores = InstructionSet(context) - self.hoistUp = InstructionSet(context) - self.sinkDown = InstructionSet(context) - self.specialHoist = InstructionSet(context) + self.toDelete = InstructionSet(context) + + self.loadsAndStores = Stack(context) + self.hoistUp = Stack(context) + self.sinkDown = Stack(context) + self.specialHoist = Stack(context) + + self.loadAndStoreAddrs = Stack(context) } mutating func deinitialize() { + toDelete.deinitialize() + loadsAndStores.deinitialize() hoistUp.deinitialize() sinkDown.deinitialize() specialHoist.deinitialize() + + loadAndStoreAddrs.deinitialize() } } @@ -48,16 +60,12 @@ private func optimizeTopLevelLoop( runsOnHighLevelSil: Bool, context: FunctionPassContext, ) { - // FIXME: Make a recursive function that populates the stack work list. Efficiency purposes. - var workList = [topLevelLoop] - var i = 0 - while i < workList.count { - let thisLoop = workList[i] - workList.append(contentsOf: thisLoop.innerLoops) - i += 1 + var workList = getWorkList(topLevelLoop: topLevelLoop, context: context) + defer { + workList.deinitialize() } - while let thisLoop = workList.popLast() { + while let thisLoop = workList.pop() { var thisLoopChanged = false repeat { @@ -74,6 +82,32 @@ private func optimizeTopLevelLoop( } } +private func getWorkList(topLevelLoop: Loop, context: Context) -> Stack { + var tmp1 = Stack(context) + var tmp2 = Stack(context) + var workList = Stack(context) + defer { + tmp1.deinitialize() + tmp2.deinitialize() + workList.deinitialize() + } + + tmp1.push(topLevelLoop) + + while !tmp1.isEmpty || !tmp2.isEmpty { + while let loop = tmp2.pop() { + workList.push(loop) + } + + while let loop = tmp1.pop() { + tmp2.push(loop) + tmp1.append(contentsOf: loop.innerLoops) + } + } + + return workList +} + private func analyzeLoop( loop: Loop, runsOnHighLevelSil: Bool, @@ -108,6 +142,11 @@ private func analyzeLoop( fullApplies.deinitialize() } + // The stack data structure doesn't keep track of element count. We need to maintain those counts manually. + var readOnlyAppliesCount = 0 + var loopSideEffectCount = 0 + var loadsCount = 0 + var hasOtherMemReadingInsts = false for bb in loop.basicBlocks { @@ -118,12 +157,14 @@ private func analyzeLoop( for inst in bb.instructions { if hasOwnershipOperandsOrResults(inst: inst) { - checkSideEffects( + if checkSideEffects( inst: inst, loopSideEffects: &loopSideEffects, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) + ) { + loopSideEffectCount += 1 + } guard let fullApply = inst as? FullApplySite else { continue } @@ -137,7 +178,8 @@ private func analyzeLoop( } case let loadInst as LoadInst: loads.push(loadInst) - movableInstructions.loadsAndStores.insert(loadInst) + loadsCount += 1 + movableInstructions.loadsAndStores.push(loadInst) case let storeInst as StoreInst: switch storeInst.storeOwnership { case .assign, .initialize: @@ -146,31 +188,37 @@ private func analyzeLoop( break } stores.push(storeInst) - movableInstructions.loadsAndStores.insert(storeInst) - checkSideEffects( + movableInstructions.loadsAndStores.push(storeInst) + if checkSideEffects( inst: storeInst, loopSideEffects: &loopSideEffects, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) + ) { + loopSideEffectCount += 1 + } case let beginAccessInst as BeginAccessInst: beginAccesses.push(beginAccessInst) - checkSideEffects( + if checkSideEffects( inst: beginAccessInst, loopSideEffects: &loopSideEffects, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) + ) { + loopSideEffectCount += 1 + } case let refElementAddrInst as RefElementAddrInst: - movableInstructions.specialHoist.insert(refElementAddrInst) + movableInstructions.specialHoist.push(refElementAddrInst) case let condFailInst as CondFailInst: - movableInstructions.hoistUp.insert(condFailInst) - checkSideEffects( + movableInstructions.hoistUp.push(condFailInst) + if checkSideEffects( inst: condFailInst, loopSideEffects: &loopSideEffects, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) + ) { + loopSideEffectCount += 1 + } case let applyInst as ApplyInst: if isSafeReadOnlyApply( applyInst: applyInst, @@ -200,49 +248,92 @@ private func analyzeLoop( default: break } - checkSideEffects( + if checkSideEffects( inst: inst, loopSideEffects: &loopSideEffects, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) + ) { + loopSideEffectCount += 1 + } - if canHoistUpDefault(inst: inst, loop: loop, domTree: context.dominatorTree) { - movableInstructions.hoistUp.insert(inst) + if canHoistUpDefault( + inst: inst, + loop: loop, + runsOnHighLevelSil: runsOnHighLevelSil, + domTree: context.dominatorTree + ) { + movableInstructions.hoistUp.push(inst) } } } } - // TODO: Check for size to avoid quadratic complexity - for readOnlyApply in readOnlyApplies { - if !mayWriteTo( - readOnlyApply, - sideEffects: loopSideEffects, - aliasAnalysis: context.aliasAnalysis, - calleeAnalysis: context.calleeAnalysis - ) { - movableInstructions.hoistUp.insert(readOnlyApply) + // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. + if readOnlyAppliesCount * loopSideEffectCount < 8000 { + for readOnlyApply in readOnlyApplies { + if !mayWriteTo( + readOnlyApply, + sideEffects: loopSideEffects, + aliasAnalysis: context.aliasAnalysis, + calleeAnalysis: context.calleeAnalysis + ) { + movableInstructions.hoistUp.push(readOnlyApply) + } } } - // TODO: Check for size to avoid quadratic complexity - for load in loads { - if !mayWriteTo(load, sideEffects: loopSideEffects, aliasAnalysis: context.aliasAnalysis) { - movableInstructions.hoistUp.insert(load) + // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. + if loadsCount * loopSideEffectCount < 8000 { + for load in loads { + if !mayWriteTo( + load, + sideEffects: loopSideEffects, + aliasAnalysis: context.aliasAnalysis + ) { + movableInstructions.hoistUp.push(load) + } } } if !globalInitCalls.isEmpty { - // TODO: Check for post dom tree root node. Check if post dominated side effects conflict with global initializer call. + // Pre check for post dom tree root node MIGHT not be necessary. + + for globalInitCall in globalInitCalls { + if !mayConflictWithGlobalInit( + globalInitCall: globalInitCall, + preheader: preheader, + sideEffects: loopSideEffects, + aliasAnalysis: context.aliasAnalysis, + postDomTree: context.postDominatorTree + ) { + movableInstructions.hoistUp.push(globalInitCall) + } + } } if !hasOtherMemReadingInsts { for storeInst in stores { let accessPath = storeInst.destination.accessPath - if accessPath.isLoopInvariant(loop: loop) { - // TODO: Add remaining checks. + if accessPath.isLoopInvariant(loop: loop), + isOnlyLoadedAndStored( + accessPath: accessPath, + storeAddr: storeInst.destination, + sideEffects: loopSideEffects, + loads: loads, + stores: stores, + aliasAnalysis: context.aliasAnalysis + ), + !movableInstructions.loadAndStoreAddrs.contains(accessPath), + splitLoads( + loads: loads, + storeAddr: storeInst.destination, + movableInstructions: movableInstructions, + accessPath: accessPath, + context: context + ) { + movableInstructions.loadAndStoreAddrs.push(accessPath) } } } @@ -254,15 +345,26 @@ private func analyzeLoop( guard fixLifetime.operand.value.type.isAddress else { continue } if sideEffectsMayRelease || !mayWriteTo(fixLifetime, sideEffects: loopSideEffects, aliasAnalysis: context.aliasAnalysis) { - movableInstructions.sinkDown.insert(fixLifetime) + movableInstructions.sinkDown.push(fixLifetime) } } } - for beginAccess in beginAccesses { - guard handledEndAccess(beginAccessInst: beginAccess, loop: loop, context: context) else { continue } - - // TODO: Analyze begin access and add instructions to the special hoist. + for beginAccessInst in beginAccesses { + if handledEndAccess( + beginAccessInst: beginAccessInst, + loop: loop, + context: context + ) && analyzeBeginAccess( + beginAccessInst: beginAccessInst, + beginAccesses: beginAccesses, + fullApplies: fullApplies, + sideEffects: loopSideEffects, + aliasAnalysis: context.aliasAnalysis, + domTree: context.dominatorTree + ){ + movableInstructions.specialHoist.push(beginAccessInst) + } } return movableInstructions @@ -276,18 +378,22 @@ private func optimizeLoop( return false } +/// Returns `true` if `inst` may have side effects. private func checkSideEffects( inst: Instruction, loopSideEffects: inout Stack, blockSideEffects: inout Stack, hasOtherMemReadingInsts: inout Bool -) { +) -> Bool { if inst.mayHaveSideEffects { loopSideEffects.push(inst) blockSideEffects.push(inst) + return true } else if inst.mayReadFromMemory { hasOtherMemReadingInsts = true } + + return false } private func hasOwnershipOperandsOrResults(inst: Instruction) -> Bool { @@ -318,6 +424,7 @@ private func isSafeReadOnlyApply( private func canHoistUpDefault( inst: Instruction, loop: Loop, + runsOnHighLevelSil: Bool, domTree: DominatorTree ) -> Bool { guard let preheader = loop.preheader else { @@ -328,8 +435,18 @@ private func canHoistUpDefault( return false } - // TODO: Handle properly the array semantics call. - // FIXME: Hoist the entire cpp thing to avoid reimplementing a lot of stuff. + switch ArraySemanticsCall.getArraySemanticsCallKind(inst: inst) { + case .getCount, .getCapacity: + if runsOnHighLevelSil && ArraySemanticsCall.canHoist(inst: inst, to: preheader.terminator, domTree: domTree) { + return true + } + case .arrayPropsIsNativeTypeChecked: + if runsOnHighLevelSil { + return false + } + default: + break + } if inst.memoryEffects == .noEffects { return true @@ -444,28 +561,215 @@ private func mayConflictWithGlobalInit( } } +private func mayConflictWithGlobalInit( + globalInitCall: Instruction, + preheader: BasicBlock, + sideEffects: Stack, + aliasAnalysis: AliasAnalysis, + postDomTree: PostDominatorTree +) -> Bool { + guard globalInitCall.parentBlock.postDominates(preheader, postDomTree) else { + return true + } + + return sideEffects + .contains { sideEffect in + globalInitCall.parentBlock.strictlyPostDominates( + sideEffect.parentBlock, + postDomTree + ) && mayConflictWithGlobalInit( + globalInitCall: globalInitCall, + sideEffect: sideEffect, + aliasAnalysis: aliasAnalysis + ) + } +} + +private func isOnlyLoadedAndStored( + accessPath: AccessPath, + storeAddr: Value, + sideEffects: Stack, + loads: Stack, + stores: Stack, + aliasAnalysis: AliasAnalysis +) -> Bool { + return !sideEffects + .contains { sideEffect in + sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) && + getStore(sideEffect, ifAccesses: accessPath) == nil && + getLoadWithAccessPath(sideEffect, ifOverlapsAccess: accessPath) == nil + } && !loads + .contains { loadInst in + loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) && + getLoadWithAccessPath(loadInst, ifOverlapsAccess: accessPath) == nil + } && !stores + .contains { storeInst in + storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && + getStore(storeInst, ifAccesses: accessPath) == nil + } +} + +typealias LoadWithAccessPath = (loadInst: LoadInst, accessPath: AccessPath) + +private func getStore( + _ inst: Instruction, + ifAccesses accessPath: AccessPath +) -> StoreInst? { + guard let storeInst = inst as? StoreInst else { + return nil + } + + // TODO: handle StoreOwnershipQualifier::Init + guard storeInst.storeOwnership != .initialize else { + return nil + } + + return accessPath == storeInst.destination.accessPath ? storeInst : nil +} + +private func isLoad( + _ inst: Instruction, + withinAccess accessPath: AccessPath +) -> Bool { + guard let loadInst = inst as? LoadInst else { + return false + } + + // TODO: Check if this is sufficient + return accessPath.getProjection(to: loadInst.address.accessPath)?.isMaterializable ?? false +} + +private func getLoadWithAccessPath( + _ inst: Instruction, + ifOverlapsAccess accessPath: AccessPath +) -> LoadWithAccessPath? { + guard let loadInst = inst as? LoadInst, + loadInst.loadOwnership != .take, // TODO: handle LoadOwnershipQualifier::Take + !loadInst.operand.value.accessPath.isEqualOrContains(accessPath), + !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) else { + return nil + } + + return (loadInst, accessPath) +} + +private func splitLoads( + loads: Stack, + storeAddr: Value, + movableInstructions: DiscoveredMovableInstructions, + accessPath: AccessPath, + context: FunctionPassContext +) -> Bool { + // TODO: Is the iterator created at the beggining of the loop immutable? + for loadInst in loads { + guard !movableInstructions.toDelete.contains(loadInst), + loadInst.mayRead(fromAddress: storeAddr, context.aliasAnalysis), + !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) else { + continue + } + + // TODO: More stuff + + loadInst.trySplit(context) + + // TODO: Rest of logic + } + + return true +} + +private func analyzeBeginAccess( + beginAccessInst: BeginAccessInst, + beginAccesses: Stack, + fullApplies: Stack, + sideEffects: Stack, + aliasAnalysis: AliasAnalysis, + domTree: DominatorTree +) -> Bool { + let areBeginAccessesSafe = beginAccesses + .allSatisfy { otherBeginAccessInst in + guard beginAccessInst != otherBeginAccessInst else { return true } + + return beginAccessInst.accessPath.isDistinct(from: otherBeginAccessInst.accessPath) + } + + guard areBeginAccessesSafe else { return false } + + for fullApplyInst in fullApplies { + guard beginAccessInst.mayWriteToMemory ? fullApplyInst.mayReadOrWrite( + address: beginAccessInst.address, + aliasAnalysis + ) : fullApplyInst.mayWrite( + toAddress: beginAccessInst.address, + aliasAnalysis + ) else { + continue + } + + if !isCoveredByScope( + beginAccessInst: beginAccessInst, + otherInst: fullApplyInst, + domTree: domTree + ) { + return false + } + } + + switch beginAccessInst.accessPath.base { + case .class, .global: + for sideEffect in sideEffects { + guard sideEffect.mayRelease else { + continue + } + + if !isCoveredByScope( + beginAccessInst: beginAccessInst, + otherInst: sideEffect, + domTree: domTree + ) { + return false + } + } + + return true + default: + return true + } +} + +private func isCoveredByScope( + beginAccessInst: BeginAccessInst, + otherInst: Instruction, + domTree: DominatorTree +) -> Bool { + return beginAccessInst.parentBlock.dominates( + otherInst.parentBlock, + domTree + ) && beginAccessInst.endAccessInstructions + .allSatisfy { endAccessInst in + otherInst.parentBlock.dominates(endAccessInst.parentBlock, domTree) + } +} + private extension AccessPath { func isLoopInvariant(loop: Loop) -> Bool { switch base { - case .argument(let functionArgument): - // TODO: Isn't function argument guaranteed to be an invariant? - if loop.basicBlockSet.contains(functionArgument.parentBlock) { - return false - } case .box(let inst as Instruction), .class(let inst as Instruction), .index(let inst as Instruction), .pointer(let inst as Instruction), .stack(let inst as Instruction), .storeBorrow(let inst as Instruction), .tail(let inst as Instruction): if loop.basicBlockSet.contains(inst.parentBlock) { return false } - case .global: + case .global, .argument: break - case .yield: - break // TODO: What is yield? + case .yield(let beginApplyResult): + if loop.basicBlockSet.contains(beginApplyResult.parentBlock) { + return false + } case .unidentified: return false } - return !projectionPath.isConstant // TODO: Is this correct? + return !projectionPath.isConstant } } diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift index 0891f83830435..9427ae2b6a4bb 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift @@ -329,7 +329,7 @@ struct FunctionPassContext : MutatingContext { var loopTree: LoopTree { let bridgedLT = _bridged.getLoopTree() - return LoopTree(bridged: bridgedLT) + return LoopTree(bridged: bridgedLT, context: self) } var swiftArrayDecl: NominalTypeDecl { diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index 80075ec9b27c9..7554441d5e65b 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -40,6 +40,7 @@ SWIFT_BEGIN_NULLABILITY_ANNOTATIONS namespace swift { class AliasAnalysis; +class ArraySemanticsCall; class BasicCalleeAnalysis; class CalleeList; class DeadEndBlocks; @@ -149,6 +150,34 @@ struct BridgedLoopTree { BRIDGED_INLINE BridgedLoop getLoop(SwiftInt index) const; }; +enum class BridgedArrayCallKind { + kNone = 0, + kArrayPropsIsNativeTypeChecked, + kCheckSubscript, + kCheckIndex, + kGetCount, + kGetCapacity, + kGetElement, + kGetElementAddress, + kMakeMutable, + kEndMutation, + kMutateUnknown, + kReserveCapacityForAppend, + kWithUnsafeMutableBufferPointer, + kAppendContentsOf, + kAppendElement, + kArrayInit, + kArrayInitEmpty, + kArrayUninitialized, + kArrayUninitializedIntrinsic, + kArrayFinalizeIntrinsic +}; + +struct BridgedArraySemanticsCall { + BRIDGED_INLINE static BridgedArrayCallKind getArraySemanticsCallKind(BridgedInstruction inst); + BRIDGED_INLINE static bool canHoist(BridgedInstruction inst, BridgedInstruction toInst, BridgedDomTree domTree); +}; + struct BridgedUtilities { typedef void (* _Nonnull VerifyFunctionFn)(BridgedPassContext, BridgedFunction); typedef void (* _Nonnull UpdateFunctionFn)(BridgedPassContext, BridgedFunction); diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index cf2d5308b740c..dc4c90aee06cc 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -21,6 +21,7 @@ #include "swift/Demangling/Demangle.h" #include "swift/SILOptimizer/Analysis/AliasAnalysis.h" +#include "swift/SILOptimizer/Analysis/ArraySemantic.h" #include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h" #include "swift/SILOptimizer/Analysis/DeadEndBlocksAnalysis.h" #include "swift/SILOptimizer/Analysis/DominanceAnalysis.h" @@ -41,6 +42,20 @@ bool BridgedAliasAnalysis::unused(BridgedValue address1, BridgedValue address2) return true; } +//===----------------------------------------------------------------------===// +// BridgedArraySemanticsCall +//===----------------------------------------------------------------------===// + +BridgedArrayCallKind BridgedArraySemanticsCall::getArraySemanticsCallKind(BridgedInstruction inst) { + ArraySemanticsCall semCall(inst.unbridged()); + return static_cast(semCall.getKind()); +} + +bool BridgedArraySemanticsCall::canHoist(BridgedInstruction inst, BridgedInstruction toInst, BridgedDomTree domTree) { + ArraySemanticsCall semCall(inst.unbridged()); + return semCall.canHoist(toInst.unbridged(), domTree.di); +} + //===----------------------------------------------------------------------===// // BridgedCalleeAnalysis //===----------------------------------------------------------------------===// From 2f8a0cd5bde921b110462a56da15877398732c62 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Thu, 17 Jul 2025 16:04:41 +0100 Subject: [PATCH 04/44] Implement the remainder of . --- .../LoopInvariantCodeMotion.swift | 80 ++++++++++++------- .../Optimizer/Utilities/OptUtils.swift | 18 +++-- 2 files changed, 64 insertions(+), 34 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 1e0336bd588dc..96e30f97a6807 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -77,7 +77,11 @@ private func optimizeTopLevelLoop( return // Encountered a loop without preheader. Return early. } - thisLoopChanged = optimizeLoop(loop: thisLoop, movableInstructions: movableInstructions) + thisLoopChanged = optimizeLoop( + loop: thisLoop, + movableInstructions: movableInstructions, + context: context + ) } while thisLoopChanged } } @@ -327,9 +331,9 @@ private func analyzeLoop( ), !movableInstructions.loadAndStoreAddrs.contains(accessPath), splitLoads( - loads: loads, + loads: &loads, storeAddr: storeInst.destination, - movableInstructions: movableInstructions, + movableInstructions: &movableInstructions, accessPath: accessPath, context: context ) { @@ -372,7 +376,8 @@ private func analyzeLoop( private func optimizeLoop( loop: Loop, - movableInstructions: DiscoveredMovableInstructions + movableInstructions: DiscoveredMovableInstructions, + context: FunctionPassContext ) -> Bool { // TODO: Optimize this loop return false @@ -596,35 +601,33 @@ private func isOnlyLoadedAndStored( return !sideEffects .contains { sideEffect in sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) && - getStore(sideEffect, ifAccesses: accessPath) == nil && - getLoadWithAccessPath(sideEffect, ifOverlapsAccess: accessPath) == nil + !isStore(sideEffect, thatAccesses: accessPath) && + !isLoadWithAccessPath(sideEffect, thatOverlapsAccess: accessPath) } && !loads .contains { loadInst in loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) && - getLoadWithAccessPath(loadInst, ifOverlapsAccess: accessPath) == nil + !isLoadWithAccessPath(loadInst, thatOverlapsAccess: accessPath) } && !stores .contains { storeInst in storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && - getStore(storeInst, ifAccesses: accessPath) == nil + !isStore(storeInst, thatAccesses: accessPath) } } -typealias LoadWithAccessPath = (loadInst: LoadInst, accessPath: AccessPath) - -private func getStore( +private func isStore( _ inst: Instruction, - ifAccesses accessPath: AccessPath -) -> StoreInst? { + thatAccesses accessPath: AccessPath +) -> Bool { guard let storeInst = inst as? StoreInst else { - return nil + return false } // TODO: handle StoreOwnershipQualifier::Init guard storeInst.storeOwnership != .initialize else { - return nil + return false } - return accessPath == storeInst.destination.accessPath ? storeInst : nil + return accessPath == storeInst.destination.accessPath } private func isLoad( @@ -639,40 +642,63 @@ private func isLoad( return accessPath.getProjection(to: loadInst.address.accessPath)?.isMaterializable ?? false } -private func getLoadWithAccessPath( +private func isLoadWithAccessPath( _ inst: Instruction, - ifOverlapsAccess accessPath: AccessPath -) -> LoadWithAccessPath? { + thatOverlapsAccess accessPath: AccessPath +) -> Bool { guard let loadInst = inst as? LoadInst, loadInst.loadOwnership != .take, // TODO: handle LoadOwnershipQualifier::Take !loadInst.operand.value.accessPath.isEqualOrContains(accessPath), !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) else { - return nil + return false } - return (loadInst, accessPath) + return true } private func splitLoads( - loads: Stack, + loads: inout Stack, storeAddr: Value, - movableInstructions: DiscoveredMovableInstructions, + movableInstructions: inout DiscoveredMovableInstructions, accessPath: AccessPath, context: FunctionPassContext ) -> Bool { + var splitCounter = 0 + // TODO: Is the iterator created at the beggining of the loop immutable? for loadInst in loads { + // TODO: What if we need to load only one field though? Then, even though a struct could have 100 fields, we would only need to load one. Right now splitLoad just splits all of the fields right? + if loadInst.type.isStruct { + guard let fields = loadInst.type.getNominalFields(in: loadInst.parentFunction), + fields.count > 0 else { + continue + } + + splitCounter += fields.count + } else if loadInst.type.isTuple { + splitCounter += loadInst.type.tupleElements.count + } else { + continue + } + + guard splitCounter <= 6 else { + return false + } + guard !movableInstructions.toDelete.contains(loadInst), loadInst.mayRead(fromAddress: storeAddr, context.aliasAnalysis), !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) else { continue } - // TODO: More stuff - - loadInst.trySplit(context) + guard loadInst.accessPath.getProjection(to: accessPath)?.isMaterializable ?? false else { + continue + } - // TODO: Rest of logic + if let splitLoads = loadInst.trySplit(context) { + loads.append(contentsOf: splitLoads) + movableInstructions.toDelete.insert(loadInst) + } } return true diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift index a7a63fe9b2cd6..07624e55201c1 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift @@ -602,14 +602,19 @@ extension StoreInst { extension LoadInst { @discardableResult func trySplit(_ context: FunctionPassContext) -> Bool { - var elements = [Value]() + return trySplit(context) != nil + } + + @discardableResult + func trySplit(_ context: FunctionPassContext) -> [LoadInst]? { + var elements = [LoadInst]() let builder = Builder(before: self, context) if type.isStruct { if (type.nominal as! StructDecl).hasUnreferenceableStorage { - return false + return nil } guard let fields = type.getNominalFields(in: parentFunction) else { - return false + return nil } for idx in 0.. LoadOwnership { From 9a3b85c3cefe092172b1ac24bcdcabdfa981b56f Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Fri, 18 Jul 2025 13:03:47 +0100 Subject: [PATCH 05/44] Add sinking and hoisting instructions. --- .../Analysis/ArraySemanticsCall.swift | 4 + .../Optimizer/Analysis/DominatorTree.swift | 8 + .../Sources/Optimizer/Analysis/LoopTree.swift | 97 +++++++++++ .../LoopInvariantCodeMotion.swift | 157 +++++++++++++++++- .../Optimizer/PassManager/Context.swift | 5 + .../Optimizer/Utilities/OptUtils.swift | 4 +- .../Sources/SIL/Instruction.swift | 4 + include/swift/SIL/SILBridging.h | 1 + include/swift/SIL/SILBridgingImpl.h | 4 + .../swift/SILOptimizer/OptimizerBridging.h | 6 + .../SILOptimizer/OptimizerBridgingImpl.h | 17 ++ 11 files changed, 304 insertions(+), 3 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift index 515bf2cf11d33..2fb54d1a15756 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift @@ -21,4 +21,8 @@ struct ArraySemanticsCall { public static func canHoist(inst: Instruction, to toInst: Instruction, domTree: DominatorTree) -> Bool { return BridgedArraySemanticsCall.canHoist(inst.bridged, toInst.bridged, domTree.bridged) } + + public static func hoist(inst: Instruction, before beforeInst: Instruction, domTree: DominatorTree) { + BridgedArraySemanticsCall.hoist(inst.bridged, beforeInst.bridged, domTree.bridged) + } } diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift index c7c106d879c08..02ed30d2265ac 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift @@ -25,4 +25,12 @@ extension BasicBlock { func strictlyDominates(_ other: BasicBlock, _ domTree: DominatorTree) -> Bool { dominates(other, domTree) && self != other } + + func isCriticalEdge(edgeIndex: Int) -> Bool { + if terminator.successors.count <= 1 && (terminator is BranchInst || terminator is CondBranchInst) { + return false + } else { + return !terminator.successors[edgeIndex].hasSinglePredecessor + } + } } diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index 1cd27c0890b2c..6f1d34b80d02d 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -25,6 +25,30 @@ struct LoopTree { self.bridged = bridged self.loops = TopLevelLoopArray(bridged, context: context) } + + func splitCriticalEdge( + basicBlock: BasicBlock, + edgeIndex: Int, + domTree: DominatorTree + ) -> BasicBlock? { + guard basicBlock.isCriticalEdge(edgeIndex: edgeIndex) else { + return nil + } + + return splitEdge( + basicBlock: basicBlock, + edgeIndex: edgeIndex, + domTree: domTree + ) + } + + func splitEdge( + basicBlock: BasicBlock, + edgeIndex: Int, + domTree: DominatorTree + ) -> BasicBlock? { + return bridged.splitEdge(basicBlock.bridged, edgeIndex, domTree.bridged).block + } } /// Describes a loop with it's children. @@ -40,6 +64,79 @@ struct Loop { bridged.getPreheader().block } + var header: BasicBlock { + bridged.getHeader().block + } + + var exitingAndLatchBlocks: Stack { + var blocks = exitingBlocks + defer { + blocks.deinitialize() + } + + for predecessor in header.predecessors { + if basicBlockSet.contains(predecessor) && !isLoopExiting(bb: predecessor) { + blocks.push(predecessor) + } + } + + return blocks + } + + var exitBlock: BasicBlock? { + var block: BasicBlock? + + // TODO: Can we do better than this? + for exit in exitBlocks { + guard block == nil else { + return nil + } + + block = exit + } + + return block + } + + var exitBlocks: Stack { + var blocks = Stack(context) + defer { + blocks.deinitialize() + } + + for bb in basicBlocks { + for succesor in bb.successors { + if !basicBlockSet.contains(bb) { + blocks.push(succesor) + } + } + } + + return blocks + } + + var exitingBlocks: Stack { + var blocks = Stack(context) + defer { + blocks.deinitialize() + } + + for bb in basicBlocks { + if isLoopExiting(bb: bb) { + blocks.push(bb) + } + } + + return blocks + } + + private func isLoopExiting(bb: BasicBlock) -> Bool { + return bb.successors + .contains { succesor in + !basicBlockSet.contains(succesor) + } + } + init(bridged: BridgedLoop, context: Context) { self.context = context self.bridged = bridged diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 96e30f97a6807..7f2e5f7c00e04 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -77,6 +77,8 @@ private func optimizeTopLevelLoop( return // Encountered a loop without preheader. Return early. } + // TODO: Would it be a good idea to convert stacks to InstructionSets so that we have more efficient lookup? + thisLoopChanged = optimizeLoop( loop: thisLoop, movableInstructions: movableInstructions, @@ -379,10 +381,161 @@ private func optimizeLoop( movableInstructions: DiscoveredMovableInstructions, context: FunctionPassContext ) -> Bool { - // TODO: Optimize this loop + guard let preheader = loop.preheader else { + return false + } + + var changed = false + + changed = hoistInstructions( + loop: loop, + hoistUp: movableInstructions.hoistUp, + context: context + ) || changed + + changed = sinkInstructions( + loop: loop, + sinkDown: movableInstructions.sinkDown, + context: context + ) || changed + + return changed +} + +func hoistInstructions( + loop: Loop, + hoistUp: Stack, + context: FunctionPassContext +) -> Bool { + guard let preheader = loop.preheader else { + return false + } + + let dominatingBlocks = getDominatingBlocks(loop: loop) + var changed = false + + for bb in dominatingBlocks { + for inst in bb.instructions { + guard hoistUp.contains(inst) else { + continue + } + + changed = hoistInstruction( + inst: inst, + preheader: preheader, + loop: loop, + context: context + ) || changed + } + } + + return changed +} + +private func hoistInstruction( + inst: Instruction, + preheader: BasicBlock, + loop: Loop, + context: FunctionPassContext +) -> Bool { + // Check whether inst is loop invariant. + guard (inst.operands.allSatisfy { operand in + !loop.basicBlockSet.contains(operand.value.parentBlock) + }) else { + return false + } + + let terminator = preheader.terminator + if ArraySemanticsCall.canHoist(inst: inst, to: terminator, domTree: context.dominatorTree) { + ArraySemanticsCall.hoist(inst: inst, before: terminator, domTree: context.dominatorTree) + } else { + inst.move(before: terminator, context) + } + return false } +private func sinkInstructions( + loop: Loop, + sinkDown: Stack, + context: FunctionPassContext +) -> Bool { + let dominatingBlocks = getDominatingBlocks(loop: loop) + var changed = false + + for inst in sinkDown { + guard dominatingBlocks.contains(inst.parentBlock) else { + continue + } + + changed = sinkInstruction( + loop: loop, + inst: inst, + context: context + ) || changed + } + + return changed +} + +private func sinkInstruction( + loop: Loop, + inst: Instruction, + context: FunctionPassContext +) -> Bool { + var exitBlock = loop.exitBlock // TODO: Change to "hasSingleExitBlock" boolean. + let exitBlocks = loop.exitBlocks + let exitingBlocks = loop.exitingBlocks + var newExitBlocks = Stack(context) + defer { + newExitBlocks.deinitialize() + } + + var changed = false + + for exitingBlock in exitingBlocks { + // TODO: On the cpp side, what's the point of copying succesors to the vector? + for (succesorIndex, succesor) in exitingBlock.successors.enumerated().reversed() where !newExitBlocks.contains(succesor) && exitBlocks.contains(succesor) { + + let outsideBlock = context.loopTree.splitCriticalEdge( + basicBlock: exitingBlock, + edgeIndex: succesorIndex, + domTree: context.dominatorTree + ) ?? succesor + + newExitBlocks.push(outsideBlock) + + if (outsideBlock.instructions.contains { otherInst in + inst.isIdenticalTo(otherInst) + }) { + exitBlock = nil + } else if let exitBlock, let firstInstruction = outsideBlock.instructions.first { + inst.move(before: firstInstruction, context) + } else if let firstInstruction = outsideBlock.instructions.first { + inst.copy(before: firstInstruction, context) + } else { + continue + } + + changed = true + } + } + + if changed && exitBlock == nil { + context.erase(instruction: inst) + } + + return changed +} + +private func getDominatingBlocks( + loop: Loop +) -> Stack { + // MARK: Implement. Wait for more efficient dom tree traversal. + + return loop.exitingAndLatchBlocks +} + /// Returns `true` if `inst` may have side effects. private func checkSideEffects( inst: Instruction, @@ -695,7 +848,7 @@ private func splitLoads( continue } - if let splitLoads = loadInst.trySplit(context) { + if let splitLoads: [LoadInst] = loadInst.trySplit(context) { loads.append(contentsOf: splitLoads) movableInstructions.toDelete.insert(loadInst) } diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift index 9427ae2b6a4bb..e874e78150b4a 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift @@ -712,6 +712,11 @@ extension Instruction { BridgedPassContext.moveInstructionBefore(bridged, otherInstruction.bridged) context.notifyInstructionsChanged() } + + func copy(before otherInstruction: Instruction, _ context: some MutatingContext) { + BridgedPassContext.copyInstructionBefore(bridged, otherInstruction.bridged) + context.notifyInstructionsChanged() + } } extension BuiltinInst { diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift index 07624e55201c1..72c59326f731e 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift @@ -602,9 +602,11 @@ extension StoreInst { extension LoadInst { @discardableResult func trySplit(_ context: FunctionPassContext) -> Bool { - return trySplit(context) != nil + let arr: [LoadInst]? = trySplit(context) + return arr != nil } + @_disfavoredOverload @discardableResult func trySplit(_ context: FunctionPassContext) -> [LoadInst]? { var elements = [LoadInst]() diff --git a/SwiftCompilerSources/Sources/SIL/Instruction.swift b/SwiftCompilerSources/Sources/SIL/Instruction.swift index 7383e91ecae50..7a059e57f52d2 100644 --- a/SwiftCompilerSources/Sources/SIL/Instruction.swift +++ b/SwiftCompilerSources/Sources/SIL/Instruction.swift @@ -163,6 +163,10 @@ public class Instruction : CustomStringConvertible, Hashable { public static func ==(lhs: Instruction, rhs: Instruction) -> Bool { lhs === rhs } + + public func isIdenticalTo(_ otherInst: Instruction) -> Bool { + return bridged.isIdenticalTo(otherInst.bridged) + } public func hash(into hasher: inout Hasher) { hasher.combine(ObjectIdentifier(self)) diff --git a/include/swift/SIL/SILBridging.h b/include/swift/SIL/SILBridging.h index d81e1f62775bf..bb3ac3f1ad201 100644 --- a/include/swift/SIL/SILBridging.h +++ b/include/swift/SIL/SILBridging.h @@ -669,6 +669,7 @@ struct BridgedInstruction { bool maySynchronize() const; bool mayBeDeinitBarrierNotConsideringSideEffects() const; BRIDGED_INLINE bool shouldBeForwarding() const; + BRIDGED_INLINE bool isIdenticalTo(BridgedInstruction inst) const; // =========================================================================// // Generalized instruction subclasses diff --git a/include/swift/SIL/SILBridgingImpl.h b/include/swift/SIL/SILBridgingImpl.h index d5dca81fbcc13..f87fce7fbdac5 100644 --- a/include/swift/SIL/SILBridgingImpl.h +++ b/include/swift/SIL/SILBridgingImpl.h @@ -1051,6 +1051,10 @@ bool BridgedInstruction::shouldBeForwarding() const { llvm::isa(unbridged()); } +bool BridgedInstruction::isIdenticalTo(BridgedInstruction inst) const { + return unbridged()->isIdenticalTo(inst.unbridged()); +} + SwiftInt BridgedInstruction::MultipleValueInstruction_getNumResults() const { return getAs()->getNumResults(); } diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index 7554441d5e65b..77d412d10eeaa 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -141,6 +141,7 @@ struct BridgedLoop { BRIDGED_INLINE BridgedBasicBlock getBasicBlock(SwiftInt index) const; BRIDGED_INLINE OptionalBridgedBasicBlock getPreheader() const; + BRIDGED_INLINE BridgedBasicBlock getHeader() const; }; struct BridgedLoopTree { @@ -148,6 +149,8 @@ struct BridgedLoopTree { BRIDGED_INLINE SwiftInt getTopLevelLoopCount() const; BRIDGED_INLINE BridgedLoop getLoop(SwiftInt index) const; + + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE OptionalBridgedBasicBlock splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const; }; enum class BridgedArrayCallKind { @@ -176,6 +179,8 @@ enum class BridgedArrayCallKind { struct BridgedArraySemanticsCall { BRIDGED_INLINE static BridgedArrayCallKind getArraySemanticsCallKind(BridgedInstruction inst); BRIDGED_INLINE static bool canHoist(BridgedInstruction inst, BridgedInstruction toInst, BridgedDomTree domTree); + + BRIDGED_INLINE static void hoist(BridgedInstruction inst, BridgedInstruction beforeInst, BridgedDomTree domTree); }; struct BridgedUtilities { @@ -291,6 +296,7 @@ struct BridgedPassContext { BRIDGED_INLINE void eraseInstruction(BridgedInstruction inst, bool salvageDebugInfo) const; BRIDGED_INLINE void eraseBlock(BridgedBasicBlock block) const; static BRIDGED_INLINE void moveInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst); + static BRIDGED_INLINE void copyInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst); bool tryOptimizeApplyOfPartialApply(BridgedInstruction closure) const; bool tryDeleteDeadClosure(BridgedInstruction closure, bool needKeepArgsAlive) const; SWIFT_IMPORT_UNSAFE DevirtResult tryDevirtualizeApply(BridgedInstruction apply, bool isMandatory) const; diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index dc4c90aee06cc..af18ff5f134d8 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -56,6 +56,11 @@ bool BridgedArraySemanticsCall::canHoist(BridgedInstruction inst, BridgedInstruc return semCall.canHoist(toInst.unbridged(), domTree.di); } +void BridgedArraySemanticsCall::hoist(BridgedInstruction inst, BridgedInstruction beforeInst, BridgedDomTree domTree) { + ArraySemanticsCall semCall(inst.unbridged()); + semCall.hoist(beforeInst.unbridged(), domTree.di); +} + //===----------------------------------------------------------------------===// // BridgedCalleeAnalysis //===----------------------------------------------------------------------===// @@ -115,6 +120,10 @@ BridgedLoop BridgedLoopTree::getLoop(SwiftInt index) const { return {li->begin()[index]}; } +OptionalBridgedBasicBlock BridgedLoopTree::splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const { + return {swift::splitEdge(bb.unbridged()->getTerminator(), edgeIndex, domTree.di, li)}; +} + SwiftInt BridgedLoop::getInnerLoopCount() const { return l->end() - l->begin(); } @@ -135,6 +144,10 @@ OptionalBridgedBasicBlock BridgedLoop::getPreheader() const { return {l->getLoopPreheader()}; } +BridgedBasicBlock BridgedLoop::getHeader() const { + return {l->getHeader()}; +} + //===----------------------------------------------------------------------===// // BridgedBasicBlockSet //===----------------------------------------------------------------------===// @@ -320,6 +333,10 @@ void BridgedPassContext::moveInstructionBefore(BridgedInstruction inst, BridgedI swift::SILBasicBlock::moveInstruction(inst.unbridged(), beforeInst.unbridged()); } +void BridgedPassContext::copyInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst) { + inst.unbridged()->clone(beforeInst.unbridged()); +} + BridgedValue BridgedPassContext::getSILUndef(BridgedType type) const { return {swift::SILUndef::get(invocation->getFunction(), type.unbridged())}; } From 27db05577e2fa765ebb2f1f598e7f54df5597ffd Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Fri, 18 Jul 2025 13:14:32 +0100 Subject: [PATCH 06/44] Format. --- .../LoopInvariantCodeMotion.swift | 488 ++++++++++-------- 1 file changed, 270 insertions(+), 218 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 7f2e5f7c00e04..3b2af35cffc14 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -12,11 +12,12 @@ import SIL -let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion") { function, context in +let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion") { + function, context in for loop in context.loopTree.loops { optimizeTopLevelLoop( topLevelLoop: loop, - runsOnHighLevelSil: true, // TODO: Make a parameter. + runsOnHighLevelSil: true, // TODO: Make a parameter. context: context ) } @@ -24,33 +25,33 @@ let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion struct DiscoveredMovableInstructions { var toDelete: InstructionSet - + var loadsAndStores: Stack var hoistUp: Stack var sinkDown: Stack var specialHoist: Stack - + var loadAndStoreAddrs: Stack - + init(context: Context) { self.toDelete = InstructionSet(context) - + self.loadsAndStores = Stack(context) self.hoistUp = Stack(context) self.sinkDown = Stack(context) self.specialHoist = Stack(context) - + self.loadAndStoreAddrs = Stack(context) } - + mutating func deinitialize() { toDelete.deinitialize() - + loadsAndStores.deinitialize() hoistUp.deinitialize() sinkDown.deinitialize() specialHoist.deinitialize() - + loadAndStoreAddrs.deinitialize() } } @@ -64,21 +65,23 @@ private func optimizeTopLevelLoop( defer { workList.deinitialize() } - + while let thisLoop = workList.pop() { var thisLoopChanged = false - + repeat { - guard let movableInstructions = analyzeLoop( - loop: thisLoop, - runsOnHighLevelSil: runsOnHighLevelSil, - context: context - ) else { - return // Encountered a loop without preheader. Return early. + guard + let movableInstructions = analyzeLoop( + loop: thisLoop, + runsOnHighLevelSil: runsOnHighLevelSil, + context: context + ) + else { + return // Encountered a loop without preheader. Return early. } - + // TODO: Would it be a good idea to convert stacks to InstructionSets so that we have more efficient lookup? - + thisLoopChanged = optimizeLoop( loop: thisLoop, movableInstructions: movableInstructions, @@ -97,20 +100,20 @@ private func getWorkList(topLevelLoop: Loop, context: Context) -> Stack { tmp2.deinitialize() workList.deinitialize() } - + tmp1.push(topLevelLoop) - + while !tmp1.isEmpty || !tmp2.isEmpty { while let loop = tmp2.pop() { workList.push(loop) } - + while let loop = tmp1.pop() { tmp2.push(loop) tmp1.append(contentsOf: loop.innerLoops) } } - + return workList } @@ -122,15 +125,15 @@ private func analyzeLoop( guard let preheader = loop.preheader else { return nil } - + var movableInstructions = DiscoveredMovableInstructions(context: context) defer { movableInstructions.deinitialize() } - + var readOnlyApplies = Stack(context) var globalInitCalls = Stack(context) - + var loopSideEffects = Stack(context) var loads = Stack(context) var stores = Stack(context) @@ -147,20 +150,20 @@ private func analyzeLoop( beginAccesses.deinitialize() fullApplies.deinitialize() } - + // The stack data structure doesn't keep track of element count. We need to maintain those counts manually. var readOnlyAppliesCount = 0 var loopSideEffectCount = 0 var loadsCount = 0 - + var hasOtherMemReadingInsts = false - + for bb in loop.basicBlocks { var blockSideEffects = Stack(context) defer { blockSideEffects.deinitialize() } - + for inst in bb.instructions { if hasOwnershipOperandsOrResults(inst: inst) { if checkSideEffects( @@ -171,12 +174,12 @@ private func analyzeLoop( ) { loopSideEffectCount += 1 } - + guard let fullApply = inst as? FullApplySite else { continue } - + fullApplies.push(fullApply) } - + switch inst { case let fixLifetimeInst as FixLifetimeInst: if fixLifetimeInst.parentBlock.dominates(preheader, context.dominatorTree) { @@ -189,7 +192,7 @@ private func analyzeLoop( case let storeInst as StoreInst: switch storeInst.storeOwnership { case .assign, .initialize: - continue // TODO: Add support + continue // TODO: Add support case .unqualified, .trivial: break } @@ -233,11 +236,14 @@ private func analyzeLoop( ) { readOnlyApplies.push(applyInst) } else if let callee = applyInst.referencedFunction, - callee.isGlobalInitFunction, - !mayConflictWithGlobalInit(globalInitCall: applyInst, sideEffects: blockSideEffects, aliasAnalysis: context.aliasAnalysis){ + callee.isGlobalInitFunction, + !mayConflictWithGlobalInit( + globalInitCall: applyInst, sideEffects: blockSideEffects, + aliasAnalysis: context.aliasAnalysis) + { globalInitCalls.push(applyInst) } - + fallthrough default: switch inst { @@ -246,14 +252,17 @@ private func analyzeLoop( case let builtinInst as BuiltinInst: switch builtinInst.id { case .Once, .OnceWithContext: - if !mayConflictWithGlobalInit(globalInitCall: builtinInst, sideEffects: blockSideEffects, aliasAnalysis: context.aliasAnalysis) { + if !mayConflictWithGlobalInit( + globalInitCall: builtinInst, sideEffects: blockSideEffects, + aliasAnalysis: context.aliasAnalysis) + { globalInitCalls.push(builtinInst) } - default: break; + default: break } default: break } - + if checkSideEffects( inst: inst, loopSideEffects: &loopSideEffects, @@ -262,7 +271,7 @@ private func analyzeLoop( ) { loopSideEffectCount += 1 } - + if canHoistUpDefault( inst: inst, loop: loop, @@ -274,7 +283,7 @@ private func analyzeLoop( } } } - + // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. if readOnlyAppliesCount * loopSideEffectCount < 8000 { for readOnlyApply in readOnlyApplies { @@ -288,7 +297,7 @@ private func analyzeLoop( } } } - + // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. if loadsCount * loopSideEffectCount < 8000 { for load in loads { @@ -301,10 +310,10 @@ private func analyzeLoop( } } } - + if !globalInitCalls.isEmpty { // Pre check for post dom tree root node MIGHT not be necessary. - + for globalInitCall in globalInitCalls { if !mayConflictWithGlobalInit( globalInitCall: globalInitCall, @@ -317,62 +326,68 @@ private func analyzeLoop( } } } - + if !hasOtherMemReadingInsts { for storeInst in stores { let accessPath = storeInst.destination.accessPath - + if accessPath.isLoopInvariant(loop: loop), - isOnlyLoadedAndStored( + isOnlyLoadedAndStored( accessPath: accessPath, storeAddr: storeInst.destination, sideEffects: loopSideEffects, loads: loads, stores: stores, aliasAnalysis: context.aliasAnalysis - ), - !movableInstructions.loadAndStoreAddrs.contains(accessPath), - splitLoads( + ), + !movableInstructions.loadAndStoreAddrs.contains(accessPath), + splitLoads( loads: &loads, storeAddr: storeInst.destination, movableInstructions: &movableInstructions, accessPath: accessPath, context: context - ) { + ) + { movableInstructions.loadAndStoreAddrs.push(accessPath) } } } - + if !fixLifetimes.isEmpty { let sideEffectsMayRelease = loopSideEffects.contains(where: { $0.mayRelease }) - + for fixLifetime in fixLifetimes { guard fixLifetime.operand.value.type.isAddress else { continue } - - if sideEffectsMayRelease || !mayWriteTo(fixLifetime, sideEffects: loopSideEffects, aliasAnalysis: context.aliasAnalysis) { + + if sideEffectsMayRelease + || !mayWriteTo( + fixLifetime, sideEffects: loopSideEffects, aliasAnalysis: context.aliasAnalysis) + { movableInstructions.sinkDown.push(fixLifetime) } } } - + for beginAccessInst in beginAccesses { if handledEndAccess( beginAccessInst: beginAccessInst, loop: loop, context: context - ) && analyzeBeginAccess( - beginAccessInst: beginAccessInst, - beginAccesses: beginAccesses, - fullApplies: fullApplies, - sideEffects: loopSideEffects, - aliasAnalysis: context.aliasAnalysis, - domTree: context.dominatorTree - ){ + ) + && analyzeBeginAccess( + beginAccessInst: beginAccessInst, + beginAccesses: beginAccesses, + fullApplies: fullApplies, + sideEffects: loopSideEffects, + aliasAnalysis: context.aliasAnalysis, + domTree: context.dominatorTree + ) + { movableInstructions.specialHoist.push(beginAccessInst) } } - + return movableInstructions } @@ -384,21 +399,23 @@ private func optimizeLoop( guard let preheader = loop.preheader else { return false } - + var changed = false - - changed = hoistInstructions( - loop: loop, - hoistUp: movableInstructions.hoistUp, - context: context - ) || changed - - changed = sinkInstructions( - loop: loop, - sinkDown: movableInstructions.sinkDown, - context: context - ) || changed - + + changed = + hoistInstructions( + loop: loop, + hoistUp: movableInstructions.hoistUp, + context: context + ) || changed + + changed = + sinkInstructions( + loop: loop, + sinkDown: movableInstructions.sinkDown, + context: context + ) || changed + return changed } @@ -410,25 +427,26 @@ func hoistInstructions( guard let preheader = loop.preheader else { return false } - + let dominatingBlocks = getDominatingBlocks(loop: loop) var changed = false - + for bb in dominatingBlocks { for inst in bb.instructions { guard hoistUp.contains(inst) else { continue } - - changed = hoistInstruction( - inst: inst, - preheader: preheader, - loop: loop, - context: context - ) || changed + + changed = + hoistInstruction( + inst: inst, + preheader: preheader, + loop: loop, + context: context + ) || changed } } - + return changed } @@ -439,19 +457,21 @@ private func hoistInstruction( context: FunctionPassContext ) -> Bool { // Check whether inst is loop invariant. - guard (inst.operands.allSatisfy { operand in - !loop.basicBlockSet.contains(operand.value.parentBlock) - }) else { + guard + (inst.operands.allSatisfy { operand in + !loop.basicBlockSet.contains(operand.value.parentBlock) + }) + else { return false } - + let terminator = preheader.terminator if ArraySemanticsCall.canHoist(inst: inst, to: terminator, domTree: context.dominatorTree) { ArraySemanticsCall.hoist(inst: inst, before: terminator, domTree: context.dominatorTree) } else { inst.move(before: terminator, context) } - + return false } @@ -462,19 +482,20 @@ private func sinkInstructions( ) -> Bool { let dominatingBlocks = getDominatingBlocks(loop: loop) var changed = false - + for inst in sinkDown { guard dominatingBlocks.contains(inst.parentBlock) else { continue } - - changed = sinkInstruction( - loop: loop, - inst: inst, - context: context - ) || changed + + changed = + sinkInstruction( + loop: loop, + inst: inst, + context: context + ) || changed } - + return changed } @@ -483,28 +504,30 @@ private func sinkInstruction( inst: Instruction, context: FunctionPassContext ) -> Bool { - var exitBlock = loop.exitBlock // TODO: Change to "hasSingleExitBlock" boolean. + var exitBlock = loop.exitBlock // TODO: Change to "hasSingleExitBlock" boolean. let exitBlocks = loop.exitBlocks let exitingBlocks = loop.exitingBlocks var newExitBlocks = Stack(context) defer { newExitBlocks.deinitialize() } - + var changed = false - + for exitingBlock in exitingBlocks { // TODO: On the cpp side, what's the point of copying succesors to the vector? - for (succesorIndex, succesor) in exitingBlock.successors.enumerated().reversed() where !newExitBlocks.contains(succesor) && exitBlocks.contains(succesor) { - - let outsideBlock = context.loopTree.splitCriticalEdge( - basicBlock: exitingBlock, - edgeIndex: succesorIndex, - domTree: context.dominatorTree - ) ?? succesor - + for (succesorIndex, succesor) in exitingBlock.successors.enumerated().reversed() + where !newExitBlocks.contains(succesor) && exitBlocks.contains(succesor) { + + let outsideBlock = + context.loopTree.splitCriticalEdge( + basicBlock: exitingBlock, + edgeIndex: succesorIndex, + domTree: context.dominatorTree + ) ?? succesor + newExitBlocks.push(outsideBlock) - + if (outsideBlock.instructions.contains { otherInst in inst.isIdenticalTo(otherInst) }) { @@ -516,15 +539,15 @@ private func sinkInstruction( } else { continue } - + changed = true } } - + if changed && exitBlock == nil { context.erase(instruction: inst) } - + return changed } @@ -532,7 +555,7 @@ private func getDominatingBlocks( loop: Loop ) -> Stack { // MARK: Implement. Wait for more efficient dom tree traversal. - + return loop.exitingAndLatchBlocks } @@ -550,15 +573,15 @@ private func checkSideEffects( } else if inst.mayReadFromMemory { hasOtherMemReadingInsts = true } - + return false } private func hasOwnershipOperandsOrResults(inst: Instruction) -> Bool { guard inst.parentFunction.hasOwnership else { return false } - - return inst.results.contains(where: { $0.ownership != .none }) || - inst.operands.contains(where: { $0.value.ownership != .none }) + + return inst.results.contains(where: { $0.ownership != .none }) + || inst.operands.contains(where: { $0.value.ownership != .none }) } private func isSafeReadOnlyApply( @@ -569,13 +592,14 @@ private func isSafeReadOnlyApply( guard applyInst.functionConvention.results.allSatisfy({ $0.convention == .unowned }) else { return false } - + if runsOnHighLevelSil, - let callee = applyInst.referencedFunction, - callee.hasSemanticsAttribute("array.props.isNativeTypeChecked") { + let callee = applyInst.referencedFunction, + callee.hasSemanticsAttribute("array.props.isNativeTypeChecked") + { return false } - + return calleeAnalysis.getSideEffects(ofApply: applyInst).isOnlyReading } @@ -588,14 +612,16 @@ private func canHoistUpDefault( guard let preheader = loop.preheader else { return false } - + if inst is TermInst || inst is Allocation || inst is Deallocation { return false } - + switch ArraySemanticsCall.getArraySemanticsCallKind(inst: inst) { case .getCount, .getCapacity: - if runsOnHighLevelSil && ArraySemanticsCall.canHoist(inst: inst, to: preheader.terminator, domTree: domTree) { + if runsOnHighLevelSil + && ArraySemanticsCall.canHoist(inst: inst, to: preheader.terminator, domTree: domTree) + { return true } case .arrayPropsIsNativeTypeChecked: @@ -605,11 +631,11 @@ private func canHoistUpDefault( default: break } - + if inst.memoryEffects == .noEffects { return true } - + return false } @@ -622,17 +648,19 @@ private func mayWriteTo( guard calleeAnalysis.getSideEffects(ofApply: applyInst).memory == .noEffects else { return false } - + for sideEffect in sideEffects { switch sideEffect { case let storeInst as StoreInst: - if storeInst.storeOwnership == .assign && - applyInst.mayRead(fromAddress: storeInst.destination, aliasAnalysis) { - return true; + if storeInst.storeOwnership == .assign + && applyInst.mayRead(fromAddress: storeInst.destination, aliasAnalysis) + { + return true } case let copyAddrInst as CopyAddrInst: - if !copyAddrInst.isInitializationOfDestination && - applyInst.mayRead(fromAddress: copyAddrInst.destination, aliasAnalysis) { + if !copyAddrInst.isInitializationOfDestination + && applyInst.mayRead(fromAddress: copyAddrInst.destination, aliasAnalysis) + { return true } case is ApplyInst, is BeginApplyInst, is TryApplyInst: @@ -640,9 +668,9 @@ private func mayWriteTo( return true } case is CondFailInst, is StrongRetainInst, is UnmanagedRetainValueInst, - is RetainValueInst, is StrongRetainUnownedInst, is FixLifetimeInst, - is KeyPathInst, is DeallocStackInst, is DeallocStackRefInst, - is DeallocRefInst: + is RetainValueInst, is StrongRetainUnownedInst, is FixLifetimeInst, + is KeyPathInst, is DeallocStackInst, is DeallocStackRefInst, + is DeallocRefInst: break default: if sideEffect.mayWriteToMemory { @@ -650,7 +678,7 @@ private func mayWriteTo( } } } - + return false } @@ -659,31 +687,38 @@ private func mayWriteTo( sideEffects: Stack, aliasAnalysis: AliasAnalysis ) -> Bool { - return sideEffects + return + sideEffects .contains { sideEffect in sideEffect.mayWrite(toAddress: unaryInst.operand.value, aliasAnalysis) } } -private func handledEndAccess(beginAccessInst: BeginAccessInst, loop: Loop, context: Context) -> Bool { +private func handledEndAccess(beginAccessInst: BeginAccessInst, loop: Loop, context: Context) + -> Bool +{ let endAccesses = getEndAccesses(beginAccessInst: beginAccessInst, context: context) - - return !endAccesses.isEmpty && !endAccesses - .contains { user in - !loop.basicBlockSet.contains(user.parentBlock) - } + + return !endAccesses.isEmpty + && !endAccesses + .contains { user in + !loop.basicBlockSet.contains(user.parentBlock) + } } -private func getEndAccesses(beginAccessInst: BeginAccessInst, context: Context) -> Stack { +private func getEndAccesses(beginAccessInst: BeginAccessInst, context: Context) -> Stack< + EndAccessInst +> { var endAccesses = Stack(context) defer { endAccesses.deinitialize() } - - endAccesses.append(contentsOf: beginAccessInst.uses.compactMap { user in - user.instruction as? EndAccessInst - }) - + + endAccesses.append( + contentsOf: beginAccessInst.uses.compactMap { user in + user.instruction as? EndAccessInst + }) + return endAccesses } @@ -709,7 +744,8 @@ private func mayConflictWithGlobalInit( sideEffects: Stack, aliasAnalysis: AliasAnalysis ) -> Bool { - return sideEffects + return + sideEffects .contains { sideEffect in mayConflictWithGlobalInit( globalInitCall: globalInitCall, @@ -729,17 +765,19 @@ private func mayConflictWithGlobalInit( guard globalInitCall.parentBlock.postDominates(preheader, postDomTree) else { return true } - - return sideEffects + + return + sideEffects .contains { sideEffect in globalInitCall.parentBlock.strictlyPostDominates( sideEffect.parentBlock, postDomTree - ) && mayConflictWithGlobalInit( - globalInitCall: globalInitCall, - sideEffect: sideEffect, - aliasAnalysis: aliasAnalysis ) + && mayConflictWithGlobalInit( + globalInitCall: globalInitCall, + sideEffect: sideEffect, + aliasAnalysis: aliasAnalysis + ) } } @@ -751,20 +789,23 @@ private func isOnlyLoadedAndStored( stores: Stack, aliasAnalysis: AliasAnalysis ) -> Bool { - return !sideEffects + return + !sideEffects .contains { sideEffect in - sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) && - !isStore(sideEffect, thatAccesses: accessPath) && - !isLoadWithAccessPath(sideEffect, thatOverlapsAccess: accessPath) - } && !loads - .contains { loadInst in - loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) && - !isLoadWithAccessPath(loadInst, thatOverlapsAccess: accessPath) - } && !stores - .contains { storeInst in - storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && - !isStore(storeInst, thatAccesses: accessPath) - } + sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) + && !isStore(sideEffect, thatAccesses: accessPath) + && !isLoadWithAccessPath(sideEffect, thatOverlapsAccess: accessPath) + } + && !loads + .contains { loadInst in + loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) + && !isLoadWithAccessPath(loadInst, thatOverlapsAccess: accessPath) + } + && !stores + .contains { storeInst in + storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) + && !isStore(storeInst, thatAccesses: accessPath) + } } private func isStore( @@ -774,12 +815,12 @@ private func isStore( guard let storeInst = inst as? StoreInst else { return false } - + // TODO: handle StoreOwnershipQualifier::Init guard storeInst.storeOwnership != .initialize else { return false } - + return accessPath == storeInst.destination.accessPath } @@ -790,7 +831,7 @@ private func isLoad( guard let loadInst = inst as? LoadInst else { return false } - + // TODO: Check if this is sufficient return accessPath.getProjection(to: loadInst.address.accessPath)?.isMaterializable ?? false } @@ -800,12 +841,13 @@ private func isLoadWithAccessPath( thatOverlapsAccess accessPath: AccessPath ) -> Bool { guard let loadInst = inst as? LoadInst, - loadInst.loadOwnership != .take, // TODO: handle LoadOwnershipQualifier::Take - !loadInst.operand.value.accessPath.isEqualOrContains(accessPath), - !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) else { + loadInst.loadOwnership != .take, // TODO: handle LoadOwnershipQualifier::Take + !loadInst.operand.value.accessPath.isEqualOrContains(accessPath), + !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) + else { return false } - + return true } @@ -817,43 +859,45 @@ private func splitLoads( context: FunctionPassContext ) -> Bool { var splitCounter = 0 - + // TODO: Is the iterator created at the beggining of the loop immutable? for loadInst in loads { // TODO: What if we need to load only one field though? Then, even though a struct could have 100 fields, we would only need to load one. Right now splitLoad just splits all of the fields right? if loadInst.type.isStruct { guard let fields = loadInst.type.getNominalFields(in: loadInst.parentFunction), - fields.count > 0 else { + fields.count > 0 + else { continue } - + splitCounter += fields.count } else if loadInst.type.isTuple { splitCounter += loadInst.type.tupleElements.count } else { continue } - + guard splitCounter <= 6 else { return false } - + guard !movableInstructions.toDelete.contains(loadInst), - loadInst.mayRead(fromAddress: storeAddr, context.aliasAnalysis), - !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) else { + loadInst.mayRead(fromAddress: storeAddr, context.aliasAnalysis), + !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) + else { continue } - + guard loadInst.accessPath.getProjection(to: accessPath)?.isMaterializable ?? false else { continue } - + if let splitLoads: [LoadInst] = loadInst.trySplit(context) { loads.append(contentsOf: splitLoads) movableInstructions.toDelete.insert(loadInst) } } - + return true } @@ -865,26 +909,31 @@ private func analyzeBeginAccess( aliasAnalysis: AliasAnalysis, domTree: DominatorTree ) -> Bool { - let areBeginAccessesSafe = beginAccesses + let areBeginAccessesSafe = + beginAccesses .allSatisfy { otherBeginAccessInst in guard beginAccessInst != otherBeginAccessInst else { return true } - + return beginAccessInst.accessPath.isDistinct(from: otherBeginAccessInst.accessPath) } - + guard areBeginAccessesSafe else { return false } - + for fullApplyInst in fullApplies { - guard beginAccessInst.mayWriteToMemory ? fullApplyInst.mayReadOrWrite( - address: beginAccessInst.address, - aliasAnalysis - ) : fullApplyInst.mayWrite( - toAddress: beginAccessInst.address, - aliasAnalysis - ) else { + guard + beginAccessInst.mayWriteToMemory + ? fullApplyInst.mayReadOrWrite( + address: beginAccessInst.address, + aliasAnalysis + ) + : fullApplyInst.mayWrite( + toAddress: beginAccessInst.address, + aliasAnalysis + ) + else { continue } - + if !isCoveredByScope( beginAccessInst: beginAccessInst, otherInst: fullApplyInst, @@ -893,14 +942,14 @@ private func analyzeBeginAccess( return false } } - + switch beginAccessInst.accessPath.base { case .class, .global: for sideEffect in sideEffects { guard sideEffect.mayRelease else { continue } - + if !isCoveredByScope( beginAccessInst: beginAccessInst, otherInst: sideEffect, @@ -909,7 +958,7 @@ private func analyzeBeginAccess( return false } } - + return true default: return true @@ -924,18 +973,21 @@ private func isCoveredByScope( return beginAccessInst.parentBlock.dominates( otherInst.parentBlock, domTree - ) && beginAccessInst.endAccessInstructions - .allSatisfy { endAccessInst in - otherInst.parentBlock.dominates(endAccessInst.parentBlock, domTree) - } + ) + && beginAccessInst.endAccessInstructions + .allSatisfy { endAccessInst in + otherInst.parentBlock.dominates(endAccessInst.parentBlock, domTree) + } } -private extension AccessPath { - func isLoopInvariant(loop: Loop) -> Bool { +extension AccessPath { + fileprivate func isLoopInvariant(loop: Loop) -> Bool { switch base { - case .box(let inst as Instruction), .class(let inst as Instruction), .index(let inst as Instruction), - .pointer(let inst as Instruction), .stack(let inst as Instruction), .storeBorrow(let inst as Instruction), - .tail(let inst as Instruction): + case .box(let inst as Instruction), .class(let inst as Instruction), + .index(let inst as Instruction), + .pointer(let inst as Instruction), .stack(let inst as Instruction), + .storeBorrow(let inst as Instruction), + .tail(let inst as Instruction): if loop.basicBlockSet.contains(inst.parentBlock) { return false } @@ -948,7 +1000,7 @@ private extension AccessPath { case .unidentified: return false } - + return !projectionPath.isConstant } } From ea81e327b514815cc75360bcde61eb3e9d3c878e Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Mon, 21 Jul 2025 12:46:05 +0100 Subject: [PATCH 07/44] Add traversal of dom children. --- .../Analysis/ArraySemanticsCall.swift | 28 ---- .../Optimizer/Analysis/DominatorTree.swift | 33 +++- .../Sources/Optimizer/Analysis/LoopTree.swift | 79 +++------ .../LoopInvariantCodeMotion.swift | 156 +++++++++++++----- .../Optimizer/PassManager/Context.swift | 13 ++ .../Sources/SIL/BasicBlock.swift | 8 + .../swift/SILOptimizer/OptimizerBridging.h | 15 +- .../SILOptimizer/OptimizerBridgingImpl.h | 44 ++--- 8 files changed, 219 insertions(+), 157 deletions(-) delete mode 100644 SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift deleted file mode 100644 index 2fb54d1a15756..0000000000000 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/ArraySemanticsCall.swift +++ /dev/null @@ -1,28 +0,0 @@ -//===--- ArraySemanticsCall.swift -----------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import SIL -import OptimizerBridging - -struct ArraySemanticsCall { - public static func getArraySemanticsCallKind(inst: Instruction) -> BridgedArrayCallKind { - return BridgedArraySemanticsCall.getArraySemanticsCallKind(inst.bridged) - } - - public static func canHoist(inst: Instruction, to toInst: Instruction, domTree: DominatorTree) -> Bool { - return BridgedArraySemanticsCall.canHoist(inst.bridged, toInst.bridged, domTree.bridged) - } - - public static func hoist(inst: Instruction, before beforeInst: Instruction, domTree: DominatorTree) { - BridgedArraySemanticsCall.hoist(inst.bridged, beforeInst.bridged, domTree.bridged) - } -} diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift index 02ed30d2265ac..7089a237ba07c 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift @@ -15,6 +15,31 @@ import OptimizerBridging struct DominatorTree { let bridged: BridgedDomTree + + func getChildren(of block: BasicBlock) -> DomChildren { + return DomChildren(bridgedDomTree: bridged, bb: block) + } +} + +struct DomChildren: BridgedRandomAccessCollection { + let bridgedDomTree: BridgedDomTree + let bb: BasicBlock + + public let count: Int + + public var startIndex: Int { return 0 } + public var endIndex: Int { return count } + + init(bridgedDomTree: BridgedDomTree, bb: BasicBlock) { + self.bridgedDomTree = bridgedDomTree + self.bb = bb + self.count = bridgedDomTree.getNumberOfChildren(bb.bridged) + } + + public subscript(_ index: Int) -> BasicBlock { + assert(index >= startIndex && index < endIndex) + return bridgedDomTree.getChildAt(bb.bridged, index).block + } } extension BasicBlock { @@ -25,12 +50,4 @@ extension BasicBlock { func strictlyDominates(_ other: BasicBlock, _ domTree: DominatorTree) -> Bool { dominates(other, domTree) && self != other } - - func isCriticalEdge(edgeIndex: Int) -> Bool { - if terminator.successors.count <= 1 && (terminator is BranchInst || terminator is CondBranchInst) { - return false - } else { - return !terminator.successors[edgeIndex].hasSinglePredecessor - } - } } diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index 6f1d34b80d02d..7c7fc664312d9 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -58,7 +58,7 @@ struct Loop { let innerLoops: LoopArray let basicBlocks: BasicBlockArray - let basicBlockSet: BasicBlockSet +// var basicBlockSet: BasicBlockSet var preheader: BasicBlock? { bridged.getPreheader().block @@ -68,72 +68,43 @@ struct Loop { bridged.getHeader().block } - var exitingAndLatchBlocks: Stack { - var blocks = exitingBlocks - defer { - blocks.deinitialize() - } - - for predecessor in header.predecessors { - if basicBlockSet.contains(predecessor) && !isLoopExiting(bb: predecessor) { - blocks.push(predecessor) + var exitingAndLatchBlocks: some Sequence { + return header.predecessors.lazy + .filter { predecessor in + basicBlocks.contains(predecessor) && !isLoopExiting(bb: predecessor) +// basicBlockSet.contains(predecessor) && !isLoopExiting(bb: predecessor) } - } - - return blocks } - var exitBlock: BasicBlock? { - var block: BasicBlock? - - // TODO: Can we do better than this? - for exit in exitBlocks { - guard block == nil else { - return nil + var exitBlocks: some Sequence { + return basicBlocks.lazy + .flatMap(\.successors) + .filter { succesor in + !basicBlocks.contains(succesor) +// !basicBlockSet.contains(succesor) } - - block = exit - } - - return block } - var exitBlocks: Stack { - var blocks = Stack(context) - defer { - blocks.deinitialize() - } - - for bb in basicBlocks { - for succesor in bb.successors { - if !basicBlockSet.contains(bb) { - blocks.push(succesor) - } + var exitingBlocks: some Sequence { + return basicBlocks.lazy + .filter { bb in + isLoopExiting(bb: bb) } - } - - return blocks } - var exitingBlocks: Stack { - var blocks = Stack(context) - defer { - blocks.deinitialize() - } - - for bb in basicBlocks { - if isLoopExiting(bb: bb) { - blocks.push(bb) - } - } - - return blocks + var isSingleExit: Bool { + return exitBlocks.singleElement != nil + } + + var hasNoExitBlocks: Bool { + return exitBlocks.isEmpty } private func isLoopExiting(bb: BasicBlock) -> Bool { return bb.successors .contains { succesor in - !basicBlockSet.contains(succesor) + !basicBlocks.contains(succesor) +// !basicBlockSet.contains(succesor) } } @@ -142,7 +113,7 @@ struct Loop { self.bridged = bridged self.innerLoops = LoopArray(bridged, context: context) self.basicBlocks = BasicBlockArray(bridged) - self.basicBlockSet = BasicBlockSet(insertContentsOf: basicBlocks, context) +// self.basicBlockSet = BasicBlockSet(insertContentsOf: basicBlocks, context) } } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 3b2af35cffc14..c47b9cc91ec15 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -82,11 +82,11 @@ private func optimizeTopLevelLoop( // TODO: Would it be a good idea to convert stacks to InstructionSets so that we have more efficient lookup? - thisLoopChanged = optimizeLoop( - loop: thisLoop, - movableInstructions: movableInstructions, - context: context - ) +// thisLoopChanged = optimizeLoop( +// loop: thisLoop, +// movableInstructions: movableInstructions, +// context: context +// ) } while thisLoopChanged } } @@ -98,7 +98,6 @@ private func getWorkList(topLevelLoop: Loop, context: Context) -> Stack { defer { tmp1.deinitialize() tmp2.deinitialize() - workList.deinitialize() } tmp1.push(topLevelLoop) @@ -276,7 +275,7 @@ private func analyzeLoop( inst: inst, loop: loop, runsOnHighLevelSil: runsOnHighLevelSil, - domTree: context.dominatorTree + context: context ) { movableInstructions.hoistUp.push(inst) } @@ -401,6 +400,8 @@ private func optimizeLoop( } var changed = false + + // MARK: Hoist all loads and stores changed = hoistInstructions( @@ -416,6 +417,13 @@ private func optimizeLoop( context: context ) || changed + changed = + hoistSpecialInstruction( + loop: loop, + specialInsts: movableInstructions.specialHoist, + context: context + ) || changed + return changed } @@ -428,7 +436,7 @@ func hoistInstructions( return false } - let dominatingBlocks = getDominatingBlocks(loop: loop) + let dominatingBlocks = getDominatingBlocks(loop: loop, context: context) var changed = false for bb in dominatingBlocks { @@ -459,15 +467,16 @@ private func hoistInstruction( // Check whether inst is loop invariant. guard (inst.operands.allSatisfy { operand in - !loop.basicBlockSet.contains(operand.value.parentBlock) + !loop.basicBlocks.contains(operand.value.parentBlock) +// !loop.basicBlockSet.contains(operand.value.parentBlock) }) else { return false } let terminator = preheader.terminator - if ArraySemanticsCall.canHoist(inst: inst, to: terminator, domTree: context.dominatorTree) { - ArraySemanticsCall.hoist(inst: inst, before: terminator, domTree: context.dominatorTree) + if inst.canHoistArraySemanticsCall(to: terminator, context) { + inst.hoistArraySemanticsCall(before: terminator, context) } else { inst.move(before: terminator, context) } @@ -480,7 +489,7 @@ private func sinkInstructions( sinkDown: Stack, context: FunctionPassContext ) -> Bool { - let dominatingBlocks = getDominatingBlocks(loop: loop) + let dominatingBlocks = getDominatingBlocks(loop: loop, context: context) var changed = false for inst in sinkDown { @@ -504,7 +513,7 @@ private func sinkInstruction( inst: Instruction, context: FunctionPassContext ) -> Bool { - var exitBlock = loop.exitBlock // TODO: Change to "hasSingleExitBlock" boolean. + var isSingleExit = loop.isSingleExit let exitBlocks = loop.exitBlocks let exitingBlocks = loop.exitingBlocks var newExitBlocks = Stack(context) @@ -515,7 +524,6 @@ private func sinkInstruction( var changed = false for exitingBlock in exitingBlocks { - // TODO: On the cpp side, what's the point of copying succesors to the vector? for (succesorIndex, succesor) in exitingBlock.successors.enumerated().reversed() where !newExitBlocks.contains(succesor) && exitBlocks.contains(succesor) { @@ -531,8 +539,8 @@ private func sinkInstruction( if (outsideBlock.instructions.contains { otherInst in inst.isIdenticalTo(otherInst) }) { - exitBlock = nil - } else if let exitBlock, let firstInstruction = outsideBlock.instructions.first { + isSingleExit = false + } else if isSingleExit, let firstInstruction = outsideBlock.instructions.first { inst.move(before: firstInstruction, context) } else if let firstInstruction = outsideBlock.instructions.first { inst.copy(before: firstInstruction, context) @@ -544,19 +552,96 @@ private func sinkInstruction( } } - if changed && exitBlock == nil { + if changed && isSingleExit { context.erase(instruction: inst) } return changed } +// TODO: Give it a better name. +private func hoistSpecialInstruction( + loop: Loop, + specialInsts: Stack, + context: FunctionPassContext +) -> Bool { + guard let preheader = loop.preheader else { return false } + + var changed = false + + for specialInst in specialInsts { + if specialInst is BeginAccessInst && loop.hasNoExitBlocks { + // TODO: We should move this as a precondition out of the loop once we remove RefElementAddrInst from here. + continue + } + + guard + hoistInstruction( + inst: specialInst, + preheader: preheader, + loop: loop, + context: context + ) + else { + continue + } + + // TODO: This should probably be moved to the hoistUp collection. We should keep special hoist to only BeginAccessInst. + if let beginAccessInst = specialInst as? BeginAccessInst { + let endAccesses = getEndAccesses(beginAccessInst: beginAccessInst, context: context) + + for endAccess in endAccesses { + _ = sinkInstruction(loop: loop, inst: endAccess, context: context) + } + } + + changed = true + } + + return changed +} + private func getDominatingBlocks( - loop: Loop + loop: Loop, + context: FunctionPassContext ) -> Stack { - // MARK: Implement. Wait for more efficient dom tree traversal. + var domBlocks = Stack(context) + defer { + domBlocks.deinitialize() + } + + getDominatingBlocksHelper( + bb: loop.header, + exitingAndLatchBBs: loop.exitingAndLatchBlocks, + domBlocks: &domBlocks, + domTree: context.dominatorTree + ) + + return domBlocks +} - return loop.exitingAndLatchBlocks +private func getDominatingBlocksHelper( + bb: BasicBlock, + exitingAndLatchBBs: some Sequence, + domBlocks: inout Stack, + domTree: DominatorTree +) { + guard exitingAndLatchBBs.allSatisfy({ exitBlock in + return bb.dominates(exitBlock, domTree) + }) else { + return + } + + domBlocks.push(bb) + + for child in domTree.getChildren(of: bb) { + getDominatingBlocksHelper( + bb: child, + exitingAndLatchBBs: exitingAndLatchBBs, + domBlocks: &domBlocks, + domTree: domTree + ) + } } /// Returns `true` if `inst` may have side effects. @@ -607,7 +692,7 @@ private func canHoistUpDefault( inst: Instruction, loop: Loop, runsOnHighLevelSil: Bool, - domTree: DominatorTree + context: FunctionPassContext ) -> Bool { guard let preheader = loop.preheader else { return false @@ -617,10 +702,10 @@ private func canHoistUpDefault( return false } - switch ArraySemanticsCall.getArraySemanticsCallKind(inst: inst) { + switch inst.getArraySemanticsCallKind() { case .getCount, .getCapacity: if runsOnHighLevelSil - && ArraySemanticsCall.canHoist(inst: inst, to: preheader.terminator, domTree: domTree) + && inst.canHoistArraySemanticsCall(to: preheader.terminator, context) { return true } @@ -702,7 +787,8 @@ private func handledEndAccess(beginAccessInst: BeginAccessInst, loop: Loop, cont return !endAccesses.isEmpty && !endAccesses .contains { user in - !loop.basicBlockSet.contains(user.parentBlock) + !loop.basicBlocks.contains(user.parentBlock) +// !loop.basicBlockSet.contains(user.parentBlock) } } @@ -862,21 +948,6 @@ private func splitLoads( // TODO: Is the iterator created at the beggining of the loop immutable? for loadInst in loads { - // TODO: What if we need to load only one field though? Then, even though a struct could have 100 fields, we would only need to load one. Right now splitLoad just splits all of the fields right? - if loadInst.type.isStruct { - guard let fields = loadInst.type.getNominalFields(in: loadInst.parentFunction), - fields.count > 0 - else { - continue - } - - splitCounter += fields.count - } else if loadInst.type.isTuple { - splitCounter += loadInst.type.tupleElements.count - } else { - continue - } - guard splitCounter <= 6 else { return false } @@ -893,6 +964,7 @@ private func splitLoads( } if let splitLoads: [LoadInst] = loadInst.trySplit(context) { + splitCounter += splitLoads.count loads.append(contentsOf: splitLoads) movableInstructions.toDelete.insert(loadInst) } @@ -988,13 +1060,15 @@ extension AccessPath { .pointer(let inst as Instruction), .stack(let inst as Instruction), .storeBorrow(let inst as Instruction), .tail(let inst as Instruction): - if loop.basicBlockSet.contains(inst.parentBlock) { +// if loop.basicBlockSet.contains(inst.parentBlock) { + if loop.basicBlocks.contains(inst.parentBlock) { return false } case .global, .argument: break case .yield(let beginApplyResult): - if loop.basicBlockSet.contains(beginApplyResult.parentBlock) { +// if loop.basicBlockSet.contains(beginApplyResult.parentBlock) { + if loop.basicBlocks.contains(beginApplyResult.parentBlock) { return false } case .unidentified: diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift index e874e78150b4a..031f26e9c4a56 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift @@ -707,6 +707,19 @@ extension Instruction { bridged.setOperand(index, value.bridged) context.notifyInstructionChanged(self) } + + func getArraySemanticsCallKind() -> BridgedArrayCallKind { + return BridgedPassContext.getArraySemanticsCallKind(self.bridged) + } + + func canHoistArraySemanticsCall(to toInst: Instruction, _ context: some MutatingContext) -> Bool { + return context._bridged.canHoistArraySemanticsCall(self.bridged, toInst.bridged) + } + + func hoistArraySemanticsCall(before toInst: Instruction, _ context: some MutatingContext) { + context._bridged.hoistArraySemanticsCall(self.bridged, toInst.bridged) // Internally updates dom tree. + context.notifyInstructionsChanged() + } func move(before otherInstruction: Instruction, _ context: some MutatingContext) { BridgedPassContext.moveInstructionBefore(bridged, otherInstruction.bridged) diff --git a/SwiftCompilerSources/Sources/SIL/BasicBlock.swift b/SwiftCompilerSources/Sources/SIL/BasicBlock.swift index 3d9613c085adc..747f5fd90ce70 100644 --- a/SwiftCompilerSources/Sources/SIL/BasicBlock.swift +++ b/SwiftCompilerSources/Sources/SIL/BasicBlock.swift @@ -85,6 +85,14 @@ final public class BasicBlock : CustomStringConvertible, HasShortDescription, Ha } fatalError() } + + public func isCriticalEdge(edgeIndex: Int) -> Bool { + if terminator.successors.count <= 1 { + return false + } else { + return !terminator.successors[edgeIndex].hasSinglePredecessor + } + } public var name: String { "bb\(index)" } diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index 77d412d10eeaa..7ecf2ac06d2fb 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -123,6 +123,8 @@ struct BridgedDomTree { swift::DominanceInfo * _Nonnull di; BRIDGED_INLINE bool dominates(BridgedBasicBlock dominating, BridgedBasicBlock dominated) const; + BRIDGED_INLINE SwiftInt getNumberOfChildren(BridgedBasicBlock bb) const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedBasicBlock getChildAt(BridgedBasicBlock bb, SwiftInt index) const; }; struct BridgedPostDomTree { @@ -176,13 +178,6 @@ enum class BridgedArrayCallKind { kArrayFinalizeIntrinsic }; -struct BridgedArraySemanticsCall { - BRIDGED_INLINE static BridgedArrayCallKind getArraySemanticsCallKind(BridgedInstruction inst); - BRIDGED_INLINE static bool canHoist(BridgedInstruction inst, BridgedInstruction toInst, BridgedDomTree domTree); - - BRIDGED_INLINE static void hoist(BridgedInstruction inst, BridgedInstruction beforeInst, BridgedDomTree domTree); -}; - struct BridgedUtilities { typedef void (* _Nonnull VerifyFunctionFn)(BridgedPassContext, BridgedFunction); typedef void (* _Nonnull UpdateFunctionFn)(BridgedPassContext, BridgedFunction); @@ -276,6 +271,12 @@ struct BridgedPassContext { SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedDeclObj getSwiftMutableSpanDecl() const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedLoopTree getLoopTree() const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedLoop getLoop() const; + + // Array semantics call + + static BRIDGED_INLINE BridgedArrayCallKind getArraySemanticsCallKind(BridgedInstruction inst); + BRIDGED_INLINE bool canHoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction toInst) const; + BRIDGED_INLINE void hoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction beforeInst) const; // AST diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index af18ff5f134d8..635129e8584e7 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -42,25 +42,6 @@ bool BridgedAliasAnalysis::unused(BridgedValue address1, BridgedValue address2) return true; } -//===----------------------------------------------------------------------===// -// BridgedArraySemanticsCall -//===----------------------------------------------------------------------===// - -BridgedArrayCallKind BridgedArraySemanticsCall::getArraySemanticsCallKind(BridgedInstruction inst) { - ArraySemanticsCall semCall(inst.unbridged()); - return static_cast(semCall.getKind()); -} - -bool BridgedArraySemanticsCall::canHoist(BridgedInstruction inst, BridgedInstruction toInst, BridgedDomTree domTree) { - ArraySemanticsCall semCall(inst.unbridged()); - return semCall.canHoist(toInst.unbridged(), domTree.di); -} - -void BridgedArraySemanticsCall::hoist(BridgedInstruction inst, BridgedInstruction beforeInst, BridgedDomTree domTree) { - ArraySemanticsCall semCall(inst.unbridged()); - semCall.hoist(beforeInst.unbridged(), domTree.di); -} - //===----------------------------------------------------------------------===// // BridgedCalleeAnalysis //===----------------------------------------------------------------------===// @@ -104,6 +85,14 @@ bool BridgedDomTree::dominates(BridgedBasicBlock dominating, BridgedBasicBlock d return di->dominates(dominating.unbridged(), dominated.unbridged()); } +SwiftInt BridgedDomTree::getNumberOfChildren(BridgedBasicBlock bb) const { + return di->getNode(bb.unbridged())->getNumChildren(); +} + +BridgedBasicBlock BridgedDomTree::getChildAt(BridgedBasicBlock bb, SwiftInt index) const { + return {di->getNode(bb.unbridged())->begin()[index]->getBlock()}; +} + bool BridgedPostDomTree::postDominates(BridgedBasicBlock dominating, BridgedBasicBlock dominated) const { return pdi->dominates(dominating.unbridged(), dominated.unbridged()); } @@ -292,6 +281,23 @@ BridgedDeclObj BridgedPassContext::getSwiftMutableSpanDecl() const { return {mod->getASTContext().getMutableSpanDecl()}; } +// Array semantics call + +BridgedArrayCallKind BridgedPassContext::getArraySemanticsCallKind(BridgedInstruction inst) { + ArraySemanticsCall semCall(inst.unbridged()); + return static_cast(semCall.getKind()); +} + +bool BridgedPassContext::canHoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction toInst) const { + ArraySemanticsCall semCall(inst.unbridged()); + return semCall.canHoist(toInst.unbridged(), getDomTree().di); +} + +void BridgedPassContext::hoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction beforeInst) const { + ArraySemanticsCall semCall(inst.unbridged()); + semCall.hoist(beforeInst.unbridged(), getDomTree().di); +} + // AST SWIFT_IMPORT_UNSAFE BRIDGED_INLINE From 45a02a6c4aa7fa4c69ac2f247c0276cd993314cd Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Mon, 21 Jul 2025 18:09:22 +0100 Subject: [PATCH 08/44] Minor bug fixes. --- .../LoopInvariantCodeMotion.swift | 114 ++++++++---------- test/SILOptimizer/licm.sil | 2 +- 2 files changed, 48 insertions(+), 68 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index c47b9cc91ec15..537504e70701b 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -24,36 +24,14 @@ let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion } struct DiscoveredMovableInstructions { - var toDelete: InstructionSet + var toDelete: Set = [] - var loadsAndStores: Stack - var hoistUp: Stack - var sinkDown: Stack - var specialHoist: Stack + var loadsAndStores: [Instruction] = [] + var hoistUp: [Instruction] = [] + var sinkDown: [Instruction] = [] + var specialHoist: [Instruction] = [] - var loadAndStoreAddrs: Stack - - init(context: Context) { - self.toDelete = InstructionSet(context) - - self.loadsAndStores = Stack(context) - self.hoistUp = Stack(context) - self.sinkDown = Stack(context) - self.specialHoist = Stack(context) - - self.loadAndStoreAddrs = Stack(context) - } - - mutating func deinitialize() { - toDelete.deinitialize() - - loadsAndStores.deinitialize() - hoistUp.deinitialize() - sinkDown.deinitialize() - specialHoist.deinitialize() - - loadAndStoreAddrs.deinitialize() - } + var loadAndStoreAddrs: [AccessPath] = [] } private func optimizeTopLevelLoop( @@ -82,11 +60,11 @@ private func optimizeTopLevelLoop( // TODO: Would it be a good idea to convert stacks to InstructionSets so that we have more efficient lookup? -// thisLoopChanged = optimizeLoop( -// loop: thisLoop, -// movableInstructions: movableInstructions, -// context: context -// ) + thisLoopChanged = optimizeLoop( + loop: thisLoop, + movableInstructions: movableInstructions, + context: context + ) } while thisLoopChanged } } @@ -125,10 +103,7 @@ private func analyzeLoop( return nil } - var movableInstructions = DiscoveredMovableInstructions(context: context) - defer { - movableInstructions.deinitialize() - } + var movableInstructions = DiscoveredMovableInstructions() var readOnlyApplies = Stack(context) var globalInitCalls = Stack(context) @@ -177,6 +152,8 @@ private func analyzeLoop( guard let fullApply = inst as? FullApplySite else { continue } fullApplies.push(fullApply) + + continue } switch inst { @@ -187,7 +164,7 @@ private func analyzeLoop( case let loadInst as LoadInst: loads.push(loadInst) loadsCount += 1 - movableInstructions.loadsAndStores.push(loadInst) + movableInstructions.loadsAndStores.append(loadInst) case let storeInst as StoreInst: switch storeInst.storeOwnership { case .assign, .initialize: @@ -196,7 +173,7 @@ private func analyzeLoop( break } stores.push(storeInst) - movableInstructions.loadsAndStores.push(storeInst) + movableInstructions.loadsAndStores.append(storeInst) if checkSideEffects( inst: storeInst, loopSideEffects: &loopSideEffects, @@ -216,9 +193,9 @@ private func analyzeLoop( loopSideEffectCount += 1 } case let refElementAddrInst as RefElementAddrInst: - movableInstructions.specialHoist.push(refElementAddrInst) + movableInstructions.specialHoist.append(refElementAddrInst) case let condFailInst as CondFailInst: - movableInstructions.hoistUp.push(condFailInst) + movableInstructions.hoistUp.append(condFailInst) if checkSideEffects( inst: condFailInst, loopSideEffects: &loopSideEffects, @@ -234,6 +211,7 @@ private func analyzeLoop( calleeAnalysis: context.calleeAnalysis ) { readOnlyApplies.push(applyInst) + readOnlyAppliesCount += 1 } else if let callee = applyInst.referencedFunction, callee.isGlobalInitFunction, !mayConflictWithGlobalInit( @@ -277,7 +255,7 @@ private func analyzeLoop( runsOnHighLevelSil: runsOnHighLevelSil, context: context ) { - movableInstructions.hoistUp.push(inst) + movableInstructions.hoistUp.append(inst) } } } @@ -292,7 +270,7 @@ private func analyzeLoop( aliasAnalysis: context.aliasAnalysis, calleeAnalysis: context.calleeAnalysis ) { - movableInstructions.hoistUp.push(readOnlyApply) + movableInstructions.hoistUp.append(readOnlyApply) } } } @@ -305,7 +283,7 @@ private func analyzeLoop( sideEffects: loopSideEffects, aliasAnalysis: context.aliasAnalysis ) { - movableInstructions.hoistUp.push(load) + movableInstructions.hoistUp.append(load) } } } @@ -321,7 +299,7 @@ private func analyzeLoop( aliasAnalysis: context.aliasAnalysis, postDomTree: context.postDominatorTree ) { - movableInstructions.hoistUp.push(globalInitCall) + movableInstructions.hoistUp.append(globalInitCall) } } } @@ -348,7 +326,7 @@ private func analyzeLoop( context: context ) { - movableInstructions.loadAndStoreAddrs.push(accessPath) + movableInstructions.loadAndStoreAddrs.append(accessPath) } } } @@ -363,7 +341,7 @@ private func analyzeLoop( || !mayWriteTo( fixLifetime, sideEffects: loopSideEffects, aliasAnalysis: context.aliasAnalysis) { - movableInstructions.sinkDown.push(fixLifetime) + movableInstructions.sinkDown.append(fixLifetime) } } } @@ -383,7 +361,7 @@ private func analyzeLoop( domTree: context.dominatorTree ) { - movableInstructions.specialHoist.push(beginAccessInst) + movableInstructions.specialHoist.append(beginAccessInst) } } @@ -429,22 +407,21 @@ private func optimizeLoop( func hoistInstructions( loop: Loop, - hoistUp: Stack, + hoistUp: [Instruction], context: FunctionPassContext ) -> Bool { guard let preheader = loop.preheader else { return false } - let dominatingBlocks = getDominatingBlocks(loop: loop, context: context) + var dominatingBlocks = getDominatingBlocks(loop: loop, context: context) + defer { + dominatingBlocks.deinitialize() + } var changed = false for bb in dominatingBlocks { - for inst in bb.instructions { - guard hoistUp.contains(inst) else { - continue - } - + for inst in bb.instructions where hoistUp.contains(inst) { changed = hoistInstruction( inst: inst, @@ -481,15 +458,18 @@ private func hoistInstruction( inst.move(before: terminator, context) } - return false + return true } private func sinkInstructions( loop: Loop, - sinkDown: Stack, + sinkDown: [Instruction], context: FunctionPassContext ) -> Bool { - let dominatingBlocks = getDominatingBlocks(loop: loop, context: context) + var dominatingBlocks = getDominatingBlocks(loop: loop, context: context) + defer { + dominatingBlocks.deinitialize() + } var changed = false for inst in sinkDown { @@ -562,7 +542,7 @@ private func sinkInstruction( // TODO: Give it a better name. private func hoistSpecialInstruction( loop: Loop, - specialInsts: Stack, + specialInsts: [Instruction], context: FunctionPassContext ) -> Bool { guard let preheader = loop.preheader else { return false } @@ -588,7 +568,10 @@ private func hoistSpecialInstruction( // TODO: This should probably be moved to the hoistUp collection. We should keep special hoist to only BeginAccessInst. if let beginAccessInst = specialInst as? BeginAccessInst { - let endAccesses = getEndAccesses(beginAccessInst: beginAccessInst, context: context) + var endAccesses = getEndAccesses(beginAccessInst: beginAccessInst, context: context) + defer { + endAccesses.deinitialize() + } for endAccess in endAccesses { _ = sinkInstruction(loop: loop, inst: endAccess, context: context) @@ -606,9 +589,6 @@ private func getDominatingBlocks( context: FunctionPassContext ) -> Stack { var domBlocks = Stack(context) - defer { - domBlocks.deinitialize() - } getDominatingBlocksHelper( bb: loop.header, @@ -782,7 +762,10 @@ private func mayWriteTo( private func handledEndAccess(beginAccessInst: BeginAccessInst, loop: Loop, context: Context) -> Bool { - let endAccesses = getEndAccesses(beginAccessInst: beginAccessInst, context: context) + var endAccesses = getEndAccesses(beginAccessInst: beginAccessInst, context: context) + defer { + endAccesses.deinitialize() + } return !endAccesses.isEmpty && !endAccesses @@ -796,9 +779,6 @@ private func getEndAccesses(beginAccessInst: BeginAccessInst, context: Context) EndAccessInst > { var endAccesses = Stack(context) - defer { - endAccesses.deinitialize() - } endAccesses.append( contentsOf: beginAccessInst.uses.compactMap { user in diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index 0dbf019064002..e149624646a1d 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=none -enable-sil-verify-all %s -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=none -enable-sil-verify-all %s -loop-invariant-code-motion | %FileCheck %s // REQUIRES: swift_in_compiler From 95b89ce90d1d8f61b8d665aac116cf9c36234b0b Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Tue, 22 Jul 2025 15:47:47 +0100 Subject: [PATCH 09/44] Minor bug fixes. --- .../Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 537504e70701b..4057c7404ea5a 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -376,11 +376,11 @@ private func optimizeLoop( guard let preheader = loop.preheader else { return false } - - var changed = false // MARK: Hoist all loads and stores + var changed = false + changed = hoistInstructions( loop: loop, @@ -532,7 +532,7 @@ private func sinkInstruction( } } - if changed && isSingleExit { + if changed && !isSingleExit { context.erase(instruction: inst) } From 12ae07b45e1e22dbc3c767544b3540bffcd5a5d6 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Thu, 24 Jul 2025 10:48:05 +0100 Subject: [PATCH 10/44] Add generalized cloner. Add initial implementation of . --- .../Sources/Optimizer/Analysis/LoopTree.swift | 2 +- .../FunctionPasses/AllocBoxToStack.swift | 5 +- .../ClosureSpecialization.swift | 269 +++++++++-------- .../InitializeStaticGlobals.swift | 6 +- .../LoopInvariantCodeMotion.swift | 279 +++++++++++++++-- .../FunctionPasses/ObjectOutliner.swift | 8 +- .../SimplifyLoad.swift | 4 +- .../Optimizer/PassManager/Context.swift | 14 +- .../Optimizer/Utilities/CMakeLists.txt | 3 +- .../Sources/Optimizer/Utilities/Cloner.swift | 159 ++++++++++ .../Utilities/SpecializationCloner.swift | 52 ---- .../Utilities/StaticInitCloner.swift | 78 ----- .../swift/SILOptimizer/OptimizerBridging.h | 21 +- lib/SILOptimizer/LoopTransforms/LICM.cpp | 1 + lib/SILOptimizer/Utils/OptimizerBridging.cpp | 139 +++++---- utils/vim/ftdetect/sil.vim | 9 - utils/vim/ftdetect/swift.vim | 9 - utils/vim/ftdetect/swiftgyb.vim | 9 - utils/vim/ftplugin/swift.vim | 21 -- utils/vim/ftplugin/swiftgyb.vim | 9 - utils/vim/syntax/sil.vim | 171 ----------- utils/vim/syntax/swift.vim | 285 ------------------ utils/vim/syntax/swiftgyb.vim | 22 -- 23 files changed, 662 insertions(+), 913 deletions(-) create mode 100644 SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift delete mode 100644 SwiftCompilerSources/Sources/Optimizer/Utilities/SpecializationCloner.swift delete mode 100644 SwiftCompilerSources/Sources/Optimizer/Utilities/StaticInitCloner.swift delete mode 100644 utils/vim/ftdetect/sil.vim delete mode 100644 utils/vim/ftdetect/swift.vim delete mode 100644 utils/vim/ftdetect/swiftgyb.vim delete mode 100644 utils/vim/ftplugin/swift.vim delete mode 100644 utils/vim/ftplugin/swiftgyb.vim delete mode 100644 utils/vim/syntax/sil.vim delete mode 100644 utils/vim/syntax/swift.vim delete mode 100644 utils/vim/syntax/swiftgyb.vim diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index 7c7fc664312d9..d3a1b491b56a6 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -73,7 +73,7 @@ struct Loop { .filter { predecessor in basicBlocks.contains(predecessor) && !isLoopExiting(bb: predecessor) // basicBlockSet.contains(predecessor) && !isLoopExiting(bb: predecessor) - } + } + exitingBlocks } var exitBlocks: some Sequence { diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift index 416350b9b8d17..9a1ae4cc8413e 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift @@ -311,10 +311,9 @@ private struct FunctionSpecializations { // This can happen if a previous run of the pass already created this specialization. return } - let cloner = SpecializationCloner(emptySpecializedFunction: specializedFunc, context) - cloner.cloneFunctionBody(from: original) - context.buildSpecializedFunction(specializedFunction: specializedFunc) { (specializedFunc, specContext) in + cloneFunction(from: original, toEmpty: specializedFunc, specContext) + replaceBoxWithStackArguments(in: specializedFunc, original: original, specContext) } context.notifyNewFunction(function: specializedFunc, derivedFrom: original) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift index ff2458dfa702b..e2cba8ce4eef1 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift @@ -25,43 +25,43 @@ /// The compiler performs reverse-mode differentiation on functions marked with `@differentiable(reverse)`. In doing so, /// it generates corresponding VJP and Pullback functions, which perform the forward and reverse pass respectively. You /// can think of VJPs as functions that "differentiate" an original function and Pullbacks as the calculated -/// "derivative" of the original function. -/// -/// VJPs always return a tuple of 2 values -- the original result and the Pullback. Pullbacks are essentially a chain +/// "derivative" of the original function. +/// +/// VJPs always return a tuple of 2 values -- the original result and the Pullback. Pullbacks are essentially a chain /// of closures, where the closure-contexts are implicitly used as the so-called "tape" during the reverse /// differentiation process. It is this chain of closures contained within the Pullbacks that this optimization aims /// to optimize via closure specialization. /// /// The code patterns that this optimization targets, look similar to the one below: /// ``` swift -/// +/// /// // Since `foo` is marked with the `differentiable(reverse)` attribute the compiler /// // will generate corresponding VJP and Pullback functions in SIL. Let's assume that /// // these functions are called `vjp_foo` and `pb_foo` respectively. -/// @differentiable(reverse) -/// func foo(_ x: Float) -> Float { +/// @differentiable(reverse) +/// func foo(_ x: Float) -> Float { /// return sin(x) /// } /// -/// //============== Before closure specialization ==============// +/// //============== Before closure specialization ==============// /// // VJP of `foo`. Returns the original result and the Pullback of `foo`. -/// sil @vjp_foo: $(Float) -> (originalResult: Float, pullback: (Float) -> Float) { -/// bb0(%0: $Float): -/// // __Inlined__ `vjp_sin`: It is important for all intermediate VJPs to have +/// sil @vjp_foo: $(Float) -> (originalResult: Float, pullback: (Float) -> Float) { +/// bb0(%0: $Float): +/// // __Inlined__ `vjp_sin`: It is important for all intermediate VJPs to have /// // been inlined in `vjp_foo`, otherwise `vjp_foo` will not be able to determine /// // that `pb_foo` is closing over other closures and no specialization will happen. -/// \ +/// \ /// %originalResult = apply @sin(%0): $(Float) -> Float \__ Inlined `vjp_sin` /// %partially_applied_pb_sin = partial_apply pb_sin(%0): $(Float) -> Float / -/// / +/// / /// /// %pb_foo = function_ref @pb_foo: $@convention(thin) (Float, (Float) -> Float) -> Float /// %partially_applied_pb_foo = partial_apply %pb_foo(%partially_applied_pb_sin): $(Float, (Float) -> Float) -> Float -/// +/// /// return (%originalResult, %partially_applied_pb_foo) /// } /// -/// // Pullback of `foo`. +/// // Pullback of `foo`. /// // /// // It receives what are called as intermediate closures that represent /// // the calculations that the Pullback needs to perform to calculate a function's @@ -70,31 +70,31 @@ /// // The intermediate closures may themselves contain intermediate closures and /// // that is why the Pullback for a function differentiated at the "top" level /// // may end up being a "chain" of closures. -/// sil @pb_foo: $(Float, (Float) -> Float) -> Float { -/// bb0(%0: $Float, %pb_sin: $(Float) -> Float): -/// %derivative_of_sin = apply %pb_sin(%0): $(Float) -> Float +/// sil @pb_foo: $(Float, (Float) -> Float) -> Float { +/// bb0(%0: $Float, %pb_sin: $(Float) -> Float): +/// %derivative_of_sin = apply %pb_sin(%0): $(Float) -> Float /// return %derivative_of_sin: Float /// } /// -/// //============== After closure specialization ==============// -/// sil @vjp_foo: $(Float) -> (originalResult: Float, pullback: (Float) -> Float) { -/// bb0(%0: $Float): -/// %originalResult = apply @sin(%0): $(Float) -> Float -/// +/// //============== After closure specialization ==============// +/// sil @vjp_foo: $(Float) -> (originalResult: Float, pullback: (Float) -> Float) { +/// bb0(%0: $Float): +/// %originalResult = apply @sin(%0): $(Float) -> Float +/// /// // Before the optimization, pullback of `foo` used to take a closure for computing /// // pullback of `sin`. Now, the specialized pullback of `foo` takes the arguments that /// // pullback of `sin` used to close over and pullback of `sin` is instead copied over /// // inside pullback of `foo`. /// %specialized_pb_foo = function_ref @specialized_pb_foo: $@convention(thin) (Float, Float) -> Float -/// %partially_applied_pb_foo = partial_apply %specialized_pb_foo(%0): $(Float, Float) -> Float -/// +/// %partially_applied_pb_foo = partial_apply %specialized_pb_foo(%0): $(Float, Float) -> Float +/// /// return (%originalResult, %partially_applied_pb_foo) /// } -/// -/// sil @specialized_pb_foo: $(Float, Float) -> Float { -/// bb0(%0: $Float, %1: $Float): -/// %2 = partial_apply @pb_sin(%1): $(Float) -> Float -/// %3 = apply %2(): $() -> Float +/// +/// sil @specialized_pb_foo: $(Float, Float) -> Float { +/// bb0(%0: $Float, %1: $Float): +/// %2 = partial_apply @pb_sin(%1): $(Float) -> Float +/// %3 = apply %2(): $() -> Float /// return %3: $Float /// } /// ``` @@ -172,25 +172,25 @@ private let specializationLevelLimit = 2 private func getPullbackClosureInfo(in caller: Function, _ context: FunctionPassContext) -> PullbackClosureInfo? { /// __Root__ closures created via `partial_apply` or `thin_to_thick_function` may be converted and reabstracted /// before finally being used at an apply site. We do not want to handle these intermediate closures separately - /// as they are handled and cloned into the specialized function as part of the root closures. Therefore, we keep - /// track of these intermediate closures in a set. - /// + /// as they are handled and cloned into the specialized function as part of the root closures. Therefore, we keep + /// track of these intermediate closures in a set. + /// /// This set is populated via the `markConvertedAndReabstractedClosuresAsUsed` function which is called when we're /// handling the different uses of our root closures. /// /// Below SIL example illustrates the above point. - /// ``` + /// ``` /// // The below set of a "root" closure and its reabstractions/conversions /// // will be handled as a unit and the entire set will be copied over - /// // in the specialized version of `takesClosure` if we determine that we + /// // in the specialized version of `takesClosure` if we determine that we /// // can specialize `takesClosure` against its closure argument. - /// __ - /// %someFunction = function_ref @someFunction: $@convention(thin) (Int, Int) -> Int \ + /// __ + /// %someFunction = function_ref @someFunction: $@convention(thin) (Int, Int) -> Int \ /// %rootClosure = partial_apply [callee_guaranteed] %someFunction (%someInt): $(Int, Int) -> Int \ - /// %thunk = function_ref @reabstractionThunk : $@convention(thin) (@callee_guaranteed (Int) -> Int) -> @out Int / - /// %reabstractedClosure = partial_apply [callee_guaranteed] %thunk(%rootClosure) : / - /// $@convention(thin) (@callee_guaranteed (Int) -> Int) -> @out Int __/ - /// + /// %thunk = function_ref @reabstractionThunk : $@convention(thin) (@callee_guaranteed (Int) -> Int) -> @out Int / + /// %reabstractedClosure = partial_apply [callee_guaranteed] %thunk(%rootClosure) : / + /// $@convention(thin) (@callee_guaranteed (Int) -> Int) -> @out Int __/ + /// /// %takesClosure = function_ref @takesClosure : $@convention(thin) (@owned @callee_guaranteed (Int) -> @out Int) -> Int /// %result = partial_apply %takesClosure(%reabstractedClosure) : $@convention(thin) (@owned @callee_guaranteed () -> @out Int) -> Int /// ret %result @@ -226,32 +226,33 @@ private func getOrCreateSpecializedFunction(basedOn pullbackClosureInfo: Pullbac let pullbackFn = pullbackClosureInfo.pullbackFn let specializedParameters = pullbackFn.convention.getSpecializedParameters(basedOn: pullbackClosureInfo) - let specializedFunction = + let specializedFunction = context.createSpecializedFunctionDeclaration(from: pullbackFn, withName: specializedFunctionName, withParams: specializedParameters, makeThin: true, makeBare: true) context.buildSpecializedFunction(specializedFunction: specializedFunction, - buildFn: { (emptySpecializedFunction, functionPassContext) in - let closureSpecCloner = SpecializationCloner(emptySpecializedFunction: emptySpecializedFunction, functionPassContext) + buildFn: { (emptySpecializedFunction, functionPassContext) in + var closureSpecCloner = Cloner(cloneToEmptyFunction: emptySpecializedFunction, functionPassContext) + defer { closureSpecCloner.deinitialize() } closureSpecCloner.cloneAndSpecializeFunctionBody(using: pullbackClosureInfo) }) return (specializedFunction, false) } -private func rewriteApplyInstruction(using specializedCallee: Function, pullbackClosureInfo: PullbackClosureInfo, +private func rewriteApplyInstruction(using specializedCallee: Function, pullbackClosureInfo: PullbackClosureInfo, _ context: FunctionPassContext) { let newApplyArgs = pullbackClosureInfo.getArgumentsForSpecializedApply(of: specializedCallee) for newApplyArg in newApplyArgs { if case let .PreviouslyCaptured(capturedArg, needsRetain, parentClosureArgIndex) = newApplyArg, - needsRetain + needsRetain { let closureArgDesc = pullbackClosureInfo.closureArgDesc(at: parentClosureArgIndex)! var builder = Builder(before: closureArgDesc.closure, context) - // TODO: Support only OSSA instructions once the OSSA elimination pass is moved after all function optimization + // TODO: Support only OSSA instructions once the OSSA elimination pass is moved after all function optimization // passes. if pullbackClosureInfo.paiOfPullback.parentBlock != closureArgDesc.closure.parentBlock { // Emit the retain and release that keeps the argument live across the callee using the closure. @@ -278,13 +279,13 @@ private func rewriteApplyInstruction(using specializedCallee: Function, pullback let funcRef = builder.createFunctionRef(specializedCallee) let capturedArgs = Array(newApplyArgs.map { $0.value }) - let newPartialApply = builder.createPartialApply(function: funcRef, substitutionMap: SubstitutionMap(), + let newPartialApply = builder.createPartialApply(function: funcRef, substitutionMap: SubstitutionMap(), capturedArguments: capturedArgs, calleeConvention: oldPartialApply.calleeConvention, hasUnknownResultIsolation: oldPartialApply.hasUnknownResultIsolation, isOnStack: oldPartialApply.isOnStack) builder = Builder(before: pullbackClosureInfo.paiOfPullback.next!, context) - // TODO: Support only OSSA instructions once the OSSA elimination pass is moved after all function optimization + // TODO: Support only OSSA instructions once the OSSA elimination pass is moved after all function optimization // passes. for closureArgDesc in pullbackClosureInfo.closureArgDescriptors { if closureArgDesc.isClosureConsumed, @@ -307,26 +308,26 @@ private func updatePullbackClosureInfo(for rootClosure: SingleValueInstruction, rootClosurePossibleLiveRange.deinitialize() } - var rootClosureApplies = OperandWorklist(context) + var rootClosureApplies = OperandWorklist(context) defer { rootClosureApplies.deinitialize() } // A "root" closure undergoing conversions and/or reabstractions has additional restrictions placed upon it, in order // for a pullback to be specialized against it. We handle conversion/reabstraction uses before we handle apply uses - // to gather the parameters required to evaluate these restrictions or to skip pullback's uses of "unsupported" + // to gather the parameters required to evaluate these restrictions or to skip pullback's uses of "unsupported" // closures altogether. // - // There are currently 2 restrictions that are evaluated prior to specializing a pullback against a converted and/or + // There are currently 2 restrictions that are evaluated prior to specializing a pullback against a converted and/or // reabstracted closure - // 1. A reabstracted root closure can only be specialized against, if the reabstracted closure is ultimately passed // trivially (as a noescape+thick function) as captured argument of pullback's partial_apply. // - // 2. A root closure may be a partial_apply [stack], in which case we need to make sure that all mark_dependence + // 2. A root closure may be a partial_apply [stack], in which case we need to make sure that all mark_dependence // bases for it will be available in the specialized callee in case the pullback is specialized against this root // closure. - let (foundUnexpectedUse, haveUsedReabstraction) = + let (foundUnexpectedUse, haveUsedReabstraction) = handleNonApplies(for: rootClosure, rootClosureApplies: &rootClosureApplies, rootClosurePossibleLiveRange: &rootClosurePossibleLiveRange, context); @@ -335,9 +336,9 @@ private func updatePullbackClosureInfo(for rootClosure: SingleValueInstruction, return } - let intermediateClosureArgDescriptorData = + let intermediateClosureArgDescriptorData = handleApplies(for: rootClosure, pullbackClosureInfoOpt: &pullbackClosureInfoOpt, rootClosureApplies: &rootClosureApplies, - rootClosurePossibleLiveRange: &rootClosurePossibleLiveRange, + rootClosurePossibleLiveRange: &rootClosurePossibleLiveRange, convertedAndReabstractedClosures: &convertedAndReabstractedClosures, haveUsedReabstraction: haveUsedReabstraction, context) @@ -352,14 +353,14 @@ private func updatePullbackClosureInfo(for rootClosure: SingleValueInstruction, /// Handles all non-apply direct and transitive uses of `rootClosure`. /// -/// Returns: -/// haveUsedReabstraction - whether the root closure is reabstracted via a thunk +/// Returns: +/// haveUsedReabstraction - whether the root closure is reabstracted via a thunk /// foundUnexpectedUse - whether the root closure is directly or transitively used in an instruction that we don't know /// how to handle. If true, then `rootClosure` should not be specialized against. -private func handleNonApplies(for rootClosure: SingleValueInstruction, +private func handleNonApplies(for rootClosure: SingleValueInstruction, rootClosureApplies: inout OperandWorklist, - rootClosurePossibleLiveRange: inout InstructionRange, - _ context: FunctionPassContext) + rootClosurePossibleLiveRange: inout InstructionRange, + _ context: FunctionPassContext) -> (foundUnexpectedUse: Bool, haveUsedReabstraction: Bool) { var foundUnexpectedUse = false @@ -392,7 +393,7 @@ private func handleNonApplies(for rootClosure: SingleValueInstruction, possibleMarkDependenceBases.deinitialize() } - var rootClosureConversionsAndReabstractions = OperandWorklist(context) + var rootClosureConversionsAndReabstractions = OperandWorklist(context) rootClosureConversionsAndReabstractions.pushIfNotVisited(contentsOf: rootClosure.uses) defer { rootClosureConversionsAndReabstractions.deinitialize() @@ -421,7 +422,7 @@ private func handleNonApplies(for rootClosure: SingleValueInstruction, pai.isSupportedClosure, pai.isPartialApplyOfThunk, // Argument must be a closure - pai.arguments[0].type.isThickFunction + pai.arguments[0].type.isThickFunction { rootClosureConversionsAndReabstractions.pushIfNotVisited(contentsOf: pai.uses) possibleMarkDependenceBases.insert(pai) @@ -437,7 +438,7 @@ private func handleNonApplies(for rootClosure: SingleValueInstruction, rootClosurePossibleLiveRange.insert(use.instruction) case let mdi as MarkDependenceInst: - if possibleMarkDependenceBases.contains(mdi.base), + if possibleMarkDependenceBases.contains(mdi.base), mdi.value == use.value, mdi.value.type.isNoEscapeFunction, mdi.value.type.isThickFunction @@ -467,7 +468,7 @@ private func handleNonApplies(for rootClosure: SingleValueInstruction, default: foundUnexpectedUse = true log("Found unexpected direct or transitive user of root closure: \(use.instruction)") - return (foundUnexpectedUse, haveUsedReabstraction) + return (foundUnexpectedUse, haveUsedReabstraction) } } @@ -477,10 +478,10 @@ private func handleNonApplies(for rootClosure: SingleValueInstruction, private typealias IntermediateClosureArgDescriptorDatum = (applySite: SingleValueInstruction, closureArgIndex: Int, paramInfo: ParameterInfo) private func handleApplies(for rootClosure: SingleValueInstruction, pullbackClosureInfoOpt: inout PullbackClosureInfo?, - rootClosureApplies: inout OperandWorklist, - rootClosurePossibleLiveRange: inout InstructionRange, - convertedAndReabstractedClosures: inout InstructionSet, haveUsedReabstraction: Bool, - _ context: FunctionPassContext) -> [IntermediateClosureArgDescriptorDatum] + rootClosureApplies: inout OperandWorklist, + rootClosurePossibleLiveRange: inout InstructionRange, + convertedAndReabstractedClosures: inout InstructionSet, haveUsedReabstraction: Bool, + _ context: FunctionPassContext) -> [IntermediateClosureArgDescriptorDatum] { var intermediateClosureArgDescriptorData: [IntermediateClosureArgDescriptorDatum] = [] @@ -541,7 +542,7 @@ private func handleApplies(for rootClosure: SingleValueInstruction, pullbackClos } // If we are going to need to release the copied over closure, we must make sure that we understand all the exit - // blocks, i.e., they terminate with an instruction that clearly indicates whether to release the copied over + // blocks, i.e., they terminate with an instruction that clearly indicates whether to release the copied over // closure or leak it. if closureParamInfo.convention.isGuaranteed, !onlyHaveThinToThickClosure, @@ -565,12 +566,12 @@ private func handleApplies(for rootClosure: SingleValueInstruction, pullbackClos // again by the ClosureSpecializer and so on. This happens if a closure argument is called _and_ referenced in // another closure, which is passed to a recursive call. E.g. // - // func foo(_ c: @escaping () -> ()) { + // func foo(_ c: @escaping () -> ()) { // c() foo({ c() }) // } // // A limit of 2 is good enough and will not be exceed in "regular" optimization scenarios. - let closureCallee = rootClosure is PartialApplyInst + let closureCallee = rootClosure is PartialApplyInst ? (rootClosure as! PartialApplyInst).referencedFunction! : (rootClosure as! ThinToThickFunctionInst).referencedFunction! @@ -579,7 +580,7 @@ private func handleApplies(for rootClosure: SingleValueInstruction, pullbackClos } if haveUsedReabstraction { - markConvertedAndReabstractedClosuresAsUsed(rootClosure: rootClosure, convertedAndReabstractedClosure: use.value, + markConvertedAndReabstractedClosuresAsUsed(rootClosure: rootClosure, convertedAndReabstractedClosure: use.value, convertedAndReabstractedClosures: &convertedAndReabstractedClosures) } @@ -609,7 +610,7 @@ private func finalizePullbackClosureInfo(for rootClosure: SingleValueInstruction if pullbackClosureInfoOpt!.paiOfPullback != applySite { fatalError("ClosureArgDescriptor's applySite field is not equal to pullback's partial_apply; got \(applySite)!") } - let closureArgDesc = ClosureArgDescriptor(closureInfo: closureInfo, closureArgumentIndex: closureArgumentIndex, + let closureArgDesc = ClosureArgDescriptor(closureInfo: closureInfo, closureArgumentIndex: closureArgumentIndex, parameterInfo: parameterInfo) pullbackClosureInfoOpt!.appendClosureArgDescriptor(closureArgDesc) } @@ -646,34 +647,34 @@ private func isClosureApplied(in callee: Function, closureArgIndex index: Int) - return inner(callee, index, &handledFuncs) } -/// Marks any converted/reabstracted closures, corresponding to a given root closure as used. We do not want to -/// look at such closures separately as during function specialization they will be handled as part of the root closure. -private func markConvertedAndReabstractedClosuresAsUsed(rootClosure: Value, convertedAndReabstractedClosure: Value, - convertedAndReabstractedClosures: inout InstructionSet) +/// Marks any converted/reabstracted closures, corresponding to a given root closure as used. We do not want to +/// look at such closures separately as during function specialization they will be handled as part of the root closure. +private func markConvertedAndReabstractedClosuresAsUsed(rootClosure: Value, convertedAndReabstractedClosure: Value, + convertedAndReabstractedClosures: inout InstructionSet) { if convertedAndReabstractedClosure != rootClosure { switch convertedAndReabstractedClosure { case let pai as PartialApplyInst: convertedAndReabstractedClosures.insert(pai) - return - markConvertedAndReabstractedClosuresAsUsed(rootClosure: rootClosure, - convertedAndReabstractedClosure: pai.arguments[0], + return + markConvertedAndReabstractedClosuresAsUsed(rootClosure: rootClosure, + convertedAndReabstractedClosure: pai.arguments[0], convertedAndReabstractedClosures: &convertedAndReabstractedClosures) case let cvt as ConvertFunctionInst: convertedAndReabstractedClosures.insert(cvt) - return - markConvertedAndReabstractedClosuresAsUsed(rootClosure: rootClosure, + return + markConvertedAndReabstractedClosuresAsUsed(rootClosure: rootClosure, convertedAndReabstractedClosure: cvt.fromFunction, convertedAndReabstractedClosures: &convertedAndReabstractedClosures) case let cvt as ConvertEscapeToNoEscapeInst: convertedAndReabstractedClosures.insert(cvt) - return - markConvertedAndReabstractedClosuresAsUsed(rootClosure: rootClosure, + return + markConvertedAndReabstractedClosuresAsUsed(rootClosure: rootClosure, convertedAndReabstractedClosure: cvt.fromFunction, convertedAndReabstractedClosures: &convertedAndReabstractedClosures) case let mdi as MarkDependenceInst: convertedAndReabstractedClosures.insert(mdi) - return + return markConvertedAndReabstractedClosuresAsUsed(rootClosure: rootClosure, convertedAndReabstractedClosure: mdi.value, convertedAndReabstractedClosures: &convertedAndReabstractedClosures) default: @@ -685,7 +686,7 @@ private func markConvertedAndReabstractedClosuresAsUsed(rootClosure: Value, conv } } -private extension SpecializationCloner { +private extension Cloner where Context == FunctionPassContext { func cloneAndSpecializeFunctionBody(using pullbackClosureInfo: PullbackClosureInfo) { self.cloneEntryBlockArgsWithoutOrigClosures(usingOrigCalleeAt: pullbackClosureInfo) @@ -699,8 +700,8 @@ private extension SpecializationCloner { private func cloneEntryBlockArgsWithoutOrigClosures(usingOrigCalleeAt pullbackClosureInfo: PullbackClosureInfo) { let originalEntryBlock = pullbackClosureInfo.pullbackFn.entryBlock - let clonedFunction = self.cloned - let clonedEntryBlock = self.entryBlock + let clonedFunction = self.targetFunction + let clonedEntryBlock = self.getOrCreateEntryBlock() originalEntryBlock.arguments .enumerated() @@ -723,12 +724,13 @@ private extension SpecializationCloner { /// of corresponding releasable closures cloned into the specialized function. We have a "list" because we clone /// "closure chains", which consist of a "root" closure and its conversions/reabstractions. This map is used to /// generate cleanup code for the cloned closures in the specialized function. - private func cloneAllClosures(at pullbackClosureInfo: PullbackClosureInfo) - -> (allSpecializedEntryBlockArgs: [Value], - closureArgIndexToAllClonedReleasableClosures: [Int: [SingleValueInstruction]]) + private func cloneAllClosures(at pullbackClosureInfo: PullbackClosureInfo) + -> (allSpecializedEntryBlockArgs: [Value], + closureArgIndexToAllClonedReleasableClosures: [Int: [SingleValueInstruction]]) { func entryBlockArgsWithOrigClosuresSkipped() -> [Value?] { - var clonedNonClosureEntryBlockArgs = self.entryBlock.arguments.makeIterator() + let clonedEntryBlock = self.getOrCreateEntryBlock() + var clonedNonClosureEntryBlockArgs = clonedEntryBlock.arguments.makeIterator() return pullbackClosureInfo.pullbackFn .entryBlock @@ -758,12 +760,12 @@ private extension SpecializationCloner { return (entryBlockArgs.map { $0! }, closureArgIndexToAllClonedReleasableClosures) } - private func cloneClosureChain(representedBy closureArgDesc: ClosureArgDescriptor, at pullbackClosureInfo: PullbackClosureInfo) - -> (finalClonedReabstractedClosure: SingleValueInstruction, allClonedReleasableClosures: [SingleValueInstruction]) + private func cloneClosureChain(representedBy closureArgDesc: ClosureArgDescriptor, at pullbackClosureInfo: PullbackClosureInfo) + -> (finalClonedReabstractedClosure: SingleValueInstruction, allClonedReleasableClosures: [SingleValueInstruction]) { let (origToClonedValueMap, capturedArgRange) = self.addEntryBlockArgs(forValuesCapturedBy: closureArgDesc) - let clonedFunction = self.cloned - let clonedEntryBlock = self.entryBlock + let clonedFunction = self.targetFunction + let clonedEntryBlock = self.getOrCreateEntryBlock() let clonedClosureArgs = Array(clonedEntryBlock.arguments[capturedArgRange]) let builder = clonedEntryBlock.instructions.isEmpty @@ -782,30 +784,30 @@ private extension SpecializationCloner { return (finalClonedReabstractedClosure, allClonedReleasableClosures) } - private func addEntryBlockArgs(forValuesCapturedBy closureArgDesc: ClosureArgDescriptor) - -> (origToClonedValueMap: [HashableValue: Value], capturedArgRange: Range) + private func addEntryBlockArgs(forValuesCapturedBy closureArgDesc: ClosureArgDescriptor) + -> (origToClonedValueMap: [HashableValue: Value], capturedArgRange: Range) { var origToClonedValueMap: [HashableValue: Value] = [:] - let clonedFunction = self.cloned - let clonedEntryBlock = self.entryBlock + let clonedFunction = self.targetFunction + let clonedEntryBlock = self.getOrCreateEntryBlock() let capturedArgRangeStart = clonedEntryBlock.arguments.count for arg in closureArgDesc.arguments { - let capturedArg = clonedEntryBlock.addFunctionArgument(type: arg.type.getLoweredType(in: clonedFunction), + let capturedArg = clonedEntryBlock.addFunctionArgument(type: arg.type.getLoweredType(in: clonedFunction), self.context) origToClonedValueMap[arg] = capturedArg } let capturedArgRangeEnd = clonedEntryBlock.arguments.count - let capturedArgRange = capturedArgRangeStart == capturedArgRangeEnd - ? 0..<0 + let capturedArgRange = capturedArgRangeStart == capturedArgRangeEnd + ? 0..<0 : capturedArgRangeStart.. SingleValueInstruction + func cloneRootClosure(representedBy closureArgDesc: ClosureArgDescriptor, capturedArguments: [Value]) + -> SingleValueInstruction { let function = self.createFunctionRef(closureArgDesc.callee) if let pai = closureArgDesc.closure as? PartialApplyInst { - return self.createPartialApply(function: function, substitutionMap: SubstitutionMap(), + return self.createPartialApply(function: function, substitutionMap: SubstitutionMap(), capturedArguments: capturedArguments, calleeConvention: pai.calleeConvention, - hasUnknownResultIsolation: pai.hasUnknownResultIsolation, + hasUnknownResultIsolation: pai.hasUnknownResultIsolation, isOnStack: pai.isOnStack) } else { return self.createThinToThickFunction(thinFunction: function, resultType: closureArgDesc.closure.type) @@ -927,10 +929,10 @@ private extension Builder { } func cloneRootClosureReabstractions(rootClosure: Value, clonedRootClosure: Value, reabstractedClosure: Value, - origToClonedValueMap: [HashableValue: Value], _ context: FunctionPassContext) + origToClonedValueMap: [HashableValue: Value], _ context: FunctionPassContext) -> SingleValueInstruction { - func inner(_ rootClosure: Value, _ clonedRootClosure: Value, _ reabstractedClosure: Value, + func inner(_ rootClosure: Value, _ clonedRootClosure: Value, _ reabstractedClosure: Value, _ origToClonedValueMap: inout [HashableValue: Value]) -> Value { switch reabstractedClosure { case let reabstractedClosure where reabstractedClosure == rootClosure: @@ -938,15 +940,15 @@ private extension Builder { return clonedRootClosure case let cvt as ConvertFunctionInst: - let toBeReabstracted = inner(rootClosure, clonedRootClosure, cvt.fromFunction, + let toBeReabstracted = inner(rootClosure, clonedRootClosure, cvt.fromFunction, &origToClonedValueMap) - let reabstracted = self.createConvertFunction(originalFunction: toBeReabstracted, resultType: cvt.type, + let reabstracted = self.createConvertFunction(originalFunction: toBeReabstracted, resultType: cvt.type, withoutActuallyEscaping: cvt.withoutActuallyEscaping) origToClonedValueMap[cvt] = reabstracted return reabstracted case let cvt as ConvertEscapeToNoEscapeInst: - let toBeReabstracted = inner(rootClosure, clonedRootClosure, cvt.fromFunction, + let toBeReabstracted = inner(rootClosure, clonedRootClosure, cvt.fromFunction, &origToClonedValueMap) let reabstracted = self.createConvertEscapeToNoEscape(originalFunction: toBeReabstracted, resultType: cvt.type, isLifetimeGuaranteed: true) @@ -954,7 +956,7 @@ private extension Builder { return reabstracted case let pai as PartialApplyInst: - let toBeReabstracted = inner(rootClosure, clonedRootClosure, pai.arguments[0], + let toBeReabstracted = inner(rootClosure, clonedRootClosure, pai.arguments[0], &origToClonedValueMap) guard let function = pai.referencedFunction else { @@ -965,10 +967,10 @@ private extension Builder { } let fri = self.createFunctionRef(function) - let reabstracted = self.createPartialApply(function: fri, substitutionMap: SubstitutionMap(), - capturedArguments: [toBeReabstracted], - calleeConvention: pai.calleeConvention, - hasUnknownResultIsolation: pai.hasUnknownResultIsolation, + let reabstracted = self.createPartialApply(function: fri, substitutionMap: SubstitutionMap(), + capturedArguments: [toBeReabstracted], + calleeConvention: pai.calleeConvention, + hasUnknownResultIsolation: pai.hasUnknownResultIsolation, isOnStack: pai.isOnStack) origToClonedValueMap[pai] = reabstracted return reabstracted @@ -989,13 +991,13 @@ private extension Builder { } var origToClonedValueMap = origToClonedValueMap - let finalClonedReabstractedClosure = inner(rootClosure, clonedRootClosure, reabstractedClosure, + let finalClonedReabstractedClosure = inner(rootClosure, clonedRootClosure, reabstractedClosure, &origToClonedValueMap) return (finalClonedReabstractedClosure as! SingleValueInstruction) } func destroyPartialApply(pai: PartialApplyInst, _ context: FunctionPassContext){ - // TODO: Support only OSSA instructions once the OSSA elimination pass is moved after all function optimization + // TODO: Support only OSSA instructions once the OSSA elimination pass is moved after all function optimization // passes. if pai.isOnStack { @@ -1040,7 +1042,7 @@ private extension FunctionConvention { // Now, append parameters captured by each of the original closure parameter. // - // Captured parameters are always appended to the function signature. If the argument type of the captured + // Captured parameters are always appended to the function signature. If the argument type of the captured // parameter in the callee is: // - direct and trivial, pass the new parameter as Direct_Unowned. // - direct and non-trivial, pass the new parameter as Direct_Owned. @@ -1076,7 +1078,7 @@ private extension ParameterInfo { ? self.convention : isArgTypeTrivial ? ArgumentConvention.directUnowned : ArgumentConvention.directOwned - return ParameterInfo(type: self.type, convention: specializedParamConvention, options: self.options, + return ParameterInfo(type: self.type, convention: specializedParamConvention, options: self.options, hasLoweredAddresses: self.hasLoweredAddresses) } @@ -1113,7 +1115,7 @@ private extension PartialApplyInst { } var isPartialApplyOfThunk: Bool { - if self.numArguments == 1, + if self.numArguments == 1, let fun = self.referencedFunction, fun.thunkKind == .reabstractionThunk || fun.thunkKind == .thunk, self.arguments[0].type.isLoweredFunction, @@ -1128,7 +1130,7 @@ private extension PartialApplyInst { var hasOnlyInoutIndirectArguments: Bool { self.argumentOperands .filter { !$0.value.type.isObject } - .allSatisfy { self.convention(of: $0)!.isInout } + .allSatisfy { self.convention(of: $0)!.isInout } } } @@ -1319,11 +1321,10 @@ private struct PullbackClosureInfo { } func specializedCalleeName(_ context: FunctionPassContext) -> String { - let closureArgs = Array(self.closureArgDescriptors.map { $0.closure }) - let closureIndices = Array(self.closureArgDescriptors.map { $0.closureArgIndex }) - - return context.mangle(withClosureArguments: closureArgs, closureArgIndices: closureIndices, - from: pullbackFn) + let closureArgs = Array(self.closureArgDescriptors.map { + (argumentIndex: $0.closureArgIndex, argumentValue: $0.closure) + }) + return context.mangle(withClosureArguments: closureArgs, from: pullbackFn) } } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift index ad8144a4c259c..aa2f33a712716 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift @@ -207,7 +207,7 @@ private indirect enum GlobalInitValue { /// Creates SIL for this global init value in the initializer of the `global`. func materialize(into global: GlobalVariable, from function: Function, _ context: FunctionPassContext) { - var cloner = StaticInitCloner(cloneTo: global, context) + var cloner = Cloner(cloneToGlobal: global, context) defer { cloner.deinitialize() } let builder = Builder(staticInitializerOf: global, context) @@ -216,7 +216,7 @@ private indirect enum GlobalInitValue { private func materializeRecursively( type: Type, - _ cloner: inout StaticInitCloner, + _ cloner: inout Cloner, _ builder: Builder, _ function: Function ) -> Value { @@ -225,7 +225,7 @@ private indirect enum GlobalInitValue { fatalError("cannot materialize undefined init value") case .constant(let value): - return cloner.clone(value) + return cloner.cloneRecursively(value: value) case .aggregate(let fields): if type.isStruct { diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 4057c7404ea5a..a15ccdb7245f2 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -373,11 +373,17 @@ private func optimizeLoop( movableInstructions: DiscoveredMovableInstructions, context: FunctionPassContext ) -> Bool { - guard let preheader = loop.preheader else { + guard loop.preheader != nil else { return false } - // MARK: Hoist all loads and stores + if hoistAllLoadsAndStores( + loop: loop, + movableInstructions: movableInstructions, + context: context + ) { + return true // TODO: This is not very legible. We should break down this function or make a separate pass out of this. + } var changed = false @@ -405,6 +411,247 @@ private func optimizeLoop( return changed } +func hoistAllLoadsAndStores( + loop: Loop, + movableInstructions: DiscoveredMovableInstructions, + context: FunctionPassContext +) -> Bool { + for accessPath in movableInstructions.loadAndStoreAddrs { + hoistLoadsAndStores( + loop: loop, + loadsAndStores: movableInstructions.loadsAndStores, + accessPath: accessPath, + context: context + ) + } + // TODO: Should we clear the data structures or have some other way of not trying to re-optimize same instructions? + + guard !movableInstructions.toDelete.isEmpty else { + return false + } + + for inst in movableInstructions.toDelete { + // TODO: Is this equivalent with the C++ code? Should we just bridge the function? + if inst.isTriviallyDead { + context.erase(instruction: inst) + } + } + // TODO: Same us further up. Should we clean the data structure? + + return true +} + +func hoistLoadsAndStores( + loop: Loop, + loadsAndStores: [Instruction], + accessPath: AccessPath, + context: FunctionPassContext +) { + let exitingAndLatchBlocks = loop.exitingAndLatchBlocks + + guard storesCommonlyDominateLoopExits( + loop: loop, + accessPath: accessPath, + exitingBlocks: exitingAndLatchBlocks, + context: context + ) else { + return + } + + for exitingOrLatchBlock in exitingAndLatchBlocks { + for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !loop.basicBlocks.contains(succesor) { + _ = context.loopTree.splitCriticalEdge( + basicBlock: exitingOrLatchBlock.terminator.parentBlock, + edgeIndex: index, + domTree: context.dominatorTree + ) + } + } + + guard let preheader = loop.preheader else { + return + } + + let builder = Builder(before: preheader.terminator, context) + var ssaUpdater: SSAUpdater? + var storeAddr: Value? + + for case let storeInst as StoreInst in loadsAndStores where isStore(storeInst, thatAccesses: accessPath) { + if let srcLoadInst = storeInst.source as? LoadInst { + guard isLoadWithAccessPath(srcLoadInst, thatOverlapsAccess: accessPath) else { + return // TODO: Do we really want't to just return without processing other instructions? + } + } + + if storeAddr == nil { + storeAddr = storeInst.destination + ssaUpdater = SSAUpdater( + function: storeAddr!.parentFunction, + type: storeAddr!.type.objectType, + ownership: storeAddr!.ownership, + context + ) + } else if storeInst.destination.type != storeAddr!.type { + return + } + + ssaUpdater?.addAvailableValue(storeInst.source, in: storeInst.parentBlock) + } + + guard let storeAddr, var ssaUpdater else { + return + } + + var cloner = Cloner(cloneBefore: preheader.terminator, context) + defer { cloner.deinitialize() } + + guard let initialAddr = (cloner.cloneUseDefChain(addr: storeAddr) { srcAddr in + srcAddr.parentBlock.dominates(preheader, context.dominatorTree) + }) else { + return + } + + let ownership: LoadInst.LoadOwnership = preheader.terminator.parentFunction.hasOwnership ? .trivial : .unqualified + + let initialLoad = builder.createLoad(fromAddress: initialAddr, ownership: ownership) + ssaUpdater.addAvailableValue(initialLoad, in: preheader) + + var currentBlock: BasicBlock? + var currentVal: Value? + + for inst in loadsAndStores { + let block = inst.parentBlock + + if block != currentBlock { + currentBlock = block + currentVal = nil + } + + if let storeInst = inst as? StoreInst, isStore(storeInst, thatAccesses: accessPath) { + currentVal = storeInst.source + context.erase(instruction: storeInst) + continue + } + + guard let loadInst = inst as? LoadInst, + isLoadWithAccessPath(loadInst, thatOverlapsAccess: accessPath) else { + continue + } + + let projectedValue = projectLoadValue( + addr: loadInst.operand.value, + accessPath: loadInst.operand.value.accessPath, + rootVal: currentVal ?? ssaUpdater.getValue(inMiddleOf: block), + rootAccessPath: accessPath, + beforeInst: loadInst, + context: context + ) + + loadInst.replace(with: projectedValue, context) + } + + for exitingOrLatchBlock in exitingAndLatchBlocks { + for succesor in exitingOrLatchBlock.successors where !loop.basicBlocks.contains(succesor) { + guard let firstInst = succesor.instructions.first else { + continue + } + + let builder = Builder(before: firstInst, context) + + let ownership: StoreInst.StoreOwnership = firstInst.parentFunction.hasOwnership ? .trivial : .unqualified + builder.createStore( + source: ssaUpdater.getValue(inMiddleOf: succesor), + destination: initialAddr, + ownership: ownership + ) + } + } + + if initialLoad.isTriviallyDead { + context.erase(instruction: initialLoad) + } +} + +func projectLoadValue( + addr: Value, + accessPath: AccessPath, + rootVal: Value, + rootAccessPath: AccessPath, + beforeInst: Instruction, + context: FunctionPassContext +) -> Value { + guard accessPath != rootAccessPath else { + return rootVal + } + + guard let projectionPath = rootAccessPath.getProjection(to: accessPath) else { + fatalError() + } + + let (kind, index, path) = projectionPath.pop() + + // MARK: Finish this function. +// if let structElementAddrInst = addr as? StructElementAddrInst { +// let builder = Builder(before: beforeInst, context) +// return builder.createStructExtract(struct: structElementAddrInst.operand.value, fieldIndex: index) +// } else if let tupleElementAddrInst = addr as? TupleElementAddrInst { +// let builder = Builder(before: beforeInst, context) +// return builder.createStructExtract(struct: tupleElementAddrInst.operand.value, fieldIndex: index) +// } + + // MARK: Implement with new access path. Wait for example from Erik. + + return rootVal +} + +func storesCommonlyDominateLoopExits( + loop: Loop, + accessPath: AccessPath, + exitingBlocks: some Sequence, + context: FunctionPassContext +) -> Bool { + var stores = BasicBlockSet(context) + var storesNotAlive = BasicBlockSet(context) + var uses = Stack(context) + defer { + stores.deinitialize() + storesNotAlive.deinitialize() + uses.deinitialize() + } + + for use in loop.header.parentFunction.instructions.flatMap(\.operands) where use.value.accessPath == accessPath { + if let user = use.instruction as? StoreInst { + stores.insert(user.parentBlock) + } + } + + guard !stores.contains(loop.header) else { + return true + } + + if let preheader = loop.preheader, + stores.contains(preheader) { + return true + } + + storesNotAlive.insert(loop.header) + + var changed = false + repeat { + changed = false + for block in loop.basicBlocks where !storesNotAlive.contains(block) && !stores.contains(block) && block.predecessors.contains(where: { storesNotAlive.contains($0) }) { + storesNotAlive.insert(block) + changed = true + } + } while changed + + for exitingBlock in exitingBlocks where !exitingBlock.successors.contains(where: { $0.terminator is UnreachableInst }) && storesNotAlive.contains(exitingBlock) { + return false + } + + return true +} + func hoistInstructions( loop: Loop, hoistUp: [Instruction], @@ -855,23 +1102,20 @@ private func isOnlyLoadedAndStored( stores: Stack, aliasAnalysis: AliasAnalysis ) -> Bool { - return - !sideEffects + return !sideEffects .contains { sideEffect in sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) && !isStore(sideEffect, thatAccesses: accessPath) && !isLoadWithAccessPath(sideEffect, thatOverlapsAccess: accessPath) + } && !loads + .contains { loadInst in + loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) + && !isLoadWithAccessPath(loadInst, thatOverlapsAccess: accessPath) + } && !stores + .contains { storeInst in + storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) + && !isStore(storeInst, thatAccesses: accessPath) } - && !loads - .contains { loadInst in - loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) - && !isLoadWithAccessPath(loadInst, thatOverlapsAccess: accessPath) - } - && !stores - .contains { storeInst in - storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) - && !isStore(storeInst, thatAccesses: accessPath) - } } private func isStore( @@ -909,7 +1153,8 @@ private func isLoadWithAccessPath( guard let loadInst = inst as? LoadInst, loadInst.loadOwnership != .take, // TODO: handle LoadOwnershipQualifier::Take !loadInst.operand.value.accessPath.isEqualOrContains(accessPath), - !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) + (!accessPath.isEqualOrContains(loadInst.operand.value.accessPath) || + !loadInst.operand.value.accessPath.isEqualOrContains(accessPath)) else { return false } @@ -1022,7 +1267,7 @@ private func isCoveredByScope( otherInst: Instruction, domTree: DominatorTree ) -> Bool { - return beginAccessInst.parentBlock.dominates( + return beginAccessInst.parentBlock.strictlyDominates( otherInst.parentBlock, domTree ) @@ -1055,6 +1300,6 @@ extension AccessPath { return false } - return !projectionPath.isConstant + return projectionPath.isConstant } } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift index ae70c9eb22787..9797b207b6e09 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift @@ -363,13 +363,13 @@ private func constructObject(of allocRef: AllocRefInstBase, inInitializerOf global: GlobalVariable, _ storesToClassFields: [StoreInst], _ storesToTailElements: [StoreInst], _ context: FunctionPassContext) { - var cloner = StaticInitCloner(cloneTo: global, context) + var cloner = Cloner(cloneToGlobal: global, context) defer { cloner.deinitialize() } // Create the initializers for the fields var objectArgs = [Value]() for store in storesToClassFields { - objectArgs.append(cloner.clone(store.source as! SingleValueInstruction)) + objectArgs.append(cloner.cloneRecursively(value: store.source as! SingleValueInstruction)) } let globalBuilder = Builder(staticInitializerOf: global, context) @@ -381,7 +381,7 @@ private func constructObject(of allocRef: AllocRefInstBase, for elementIdx in 0.. String { - closureArgs.withBridgedValues { bridgedClosureArgsRef in - closureArgIndices.withBridgedArrayRef{bridgedClosureArgIndicesRef in - String(taking: _bridged.mangleWithClosureArgs( - bridgedClosureArgsRef, - bridgedClosureArgIndicesRef, - applySiteCallee.bridged - )) - } + func mangle(withClosureArguments closureArgs: [(argumentIndex: Int, argumentValue: Value)], + from applySiteCallee: Function + ) -> String { + closureArgs.withBridgedArrayRef{ bridgedClosureArgs in + String(taking: _bridged.mangleWithClosureArgs(bridgedClosureArgs, applySiteCallee.bridged)) } } diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt index ab05966d6195f..2222de4c8f637 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt @@ -10,7 +10,7 @@ swift_compiler_sources(Optimizer AccessUtilsTest.swift AddressUtils.swift BorrowUtils.swift - SpecializationCloner.swift + Cloner.swift Devirtualization.swift EscapeUtils.swift ForwardingUtils.swift @@ -22,7 +22,6 @@ swift_compiler_sources(Optimizer OwnershipLiveness.swift PhiUpdater.swift SSAUpdater.swift - StaticInitCloner.swift Test.swift Verifier.swift ) diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift new file mode 100644 index 0000000000000..72910156a22d6 --- /dev/null +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift @@ -0,0 +1,159 @@ +//===--- Cloner.swift ------------------------------------------------------==// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import OptimizerBridging +import SIL + +/// Clones the initializer value of a GlobalVariable. +/// +/// Used to transitively clone "constant" instructions, including their operands, +/// from or to the static initializer value of a GlobalVariable. +/// +struct Cloner { + private var bridged: BridgedCloner + let context: Context + + enum Target { + case function(Function) + case global(GlobalVariable) + } + let target: Target + + init(cloneToGlobal: GlobalVariable, _ context: Context) { + self.bridged = BridgedCloner(cloneToGlobal.bridged, context._bridged) + self.context = context + self.target = .global(cloneToGlobal) + } + + init(cloneBefore inst: Instruction, _ context: Context) { + self.bridged = BridgedCloner(inst.bridged, context._bridged) + self.context = context + self.target = .function(inst.parentFunction) + } + + init(cloneToEmptyFunction: Function, _ context: Context) where Context == FunctionPassContext { + self.bridged = BridgedCloner(cloneToEmptyFunction.bridged, context._bridged) + self.context = context + self.target = .function(cloneToEmptyFunction) + } + + mutating func deinitialize() { + bridged.destroy(context._bridged) + } + + var targetFunction: Function { + guard case .function(let function) = target else { + fatalError("expected cloning into a function") + } + return function + } + + mutating func clone(instruction: Instruction) -> Instruction { + let cloned = bridged.clone(instruction.bridged).instruction + if case .function = target { + context.notifyInstructionChanged(cloned) + context.notifyInstructionsChanged() + } + return cloned + } + + /// Transitively clones `value` including its defining instruction's operands. + mutating func cloneRecursively(value: Value) -> Value { + if isCloned(value: value) { + return getClonedValue(of: value) + } + + guard let inst = value.definingInstruction else { + fatalError("expected instruction to clone or already cloned value") + } + + for op in inst.operands { + _ = cloneRecursively(value: op.value) + } + + let cloned = clone(instruction: inst) + if let svi = cloned as? SingleValueInstruction { + return svi + } else if let originalMvi = value as? MultipleValueInstructionResult { + return cloned.results[originalMvi.index] + } + fatalError("unexpected instruction kind") + } + + mutating func cloneUseDefChain(addr: Value, checkBase: (Value) -> Bool) -> Value? { + guard !checkBase(addr) else { + return addr + } + + switch addr { + // The cloner does not currently know how to create compensating + // end_borrows or fix mark_dependence operands. + case is BeginBorrowInst, is MarkDependenceInst: return nil + case let singleValueInstruction as SingleValueInstruction: + // TODO: Double check whether correct + guard let sourceOperand = singleValueInstruction.operands.first else { return nil } + + return cloneProjection(projectAddr: singleValueInstruction, sourceOperand: sourceOperand) + default: return nil + } + } + + private mutating func cloneProjection( + projectAddr: SingleValueInstruction, + sourceOperand: Operand + ) -> Value { + let projectedSource = cloneRecursively(value: sourceOperand.value) + + let clone = clone(instruction: projectAddr) + clone.setOperand(at: sourceOperand.index, to: projectedSource, context) + return clone as! SingleValueInstruction + } + + mutating func getClonedValue(of originalValue: Value) -> Value { + bridged.getClonedValue(originalValue.bridged).value + } + + func isCloned(value: Value) -> Bool { + bridged.isValueCloned(value.bridged) + } + + func getClonedBlock(for originalBlock: BasicBlock) -> BasicBlock { + bridged.getClonedBasicBlock(originalBlock.bridged).block + } + +} + +extension Cloner where Context == FunctionPassContext { + func getOrCreateEntryBlock() -> BasicBlock { + if let entryBlock = targetFunction.blocks.first { + return entryBlock + } + return targetFunction.appendNewBlock(context) + } + + func cloneFunctionBody(from originalFunction: Function, entryBlockArguments: [Value]) { + entryBlockArguments.withBridgedValues { bridgedEntryBlockArgs in + let entryBlock = getOrCreateEntryBlock() + bridged.cloneFunctionBody(originalFunction.bridged, entryBlock.bridged, bridgedEntryBlockArgs) + } + } + + func cloneFunctionBody(from originalFunction: Function) { + bridged.cloneFunctionBody(originalFunction.bridged) + } +} + +func cloneFunction(from originalFunction: Function, toEmpty targetFunction: Function, _ context: FunctionPassContext) { + var cloner = Cloner(cloneToEmptyFunction: targetFunction, context) + defer { cloner.deinitialize() } + cloner.cloneFunctionBody(from: originalFunction) +} diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/SpecializationCloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/SpecializationCloner.swift deleted file mode 100644 index 9aaf5f3b09c98..0000000000000 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/SpecializationCloner.swift +++ /dev/null @@ -1,52 +0,0 @@ -//===--- SpecializationCloner.swift --------------------------------------------==// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import OptimizerBridging -import SIL - -/// Utility cloner type that can be used by optimizations that generate new functions or specialized versions of -/// existing functions. -struct SpecializationCloner { - private let bridged: BridgedSpecializationCloner - let context: FunctionPassContext - - init(emptySpecializedFunction: Function, _ context: FunctionPassContext) { - self.bridged = BridgedSpecializationCloner(emptySpecializedFunction.bridged) - self.context = context - } - - var cloned: Function { - bridged.getCloned().function - } - - var entryBlock: BasicBlock { - if cloned.blocks.isEmpty { - return cloned.appendNewBlock(context) - } else { - return cloned.entryBlock - } - } - - func getClonedBlock(for originalBlock: BasicBlock) -> BasicBlock { - bridged.getClonedBasicBlock(originalBlock.bridged).block - } - - func cloneFunctionBody(from originalFunction: Function, entryBlockArguments: [Value]) { - entryBlockArguments.withBridgedValues { bridgedEntryBlockArgs in - bridged.cloneFunctionBody(originalFunction.bridged, self.entryBlock.bridged, bridgedEntryBlockArgs) - } - } - - func cloneFunctionBody(from originalFunction: Function) { - bridged.cloneFunctionBody(originalFunction.bridged) - } -} diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/StaticInitCloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/StaticInitCloner.swift deleted file mode 100644 index dcc9853052c6a..0000000000000 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/StaticInitCloner.swift +++ /dev/null @@ -1,78 +0,0 @@ -//===--- StaticInitCloner.swift --------------------------------------------==// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import SIL -import OptimizerBridging - -/// Clones the initializer value of a GlobalVariable. -/// -/// Used to transitively clone "constant" instructions, including their operands, -/// from or to the static initializer value of a GlobalVariable. -/// -struct StaticInitCloner { - private var bridged: BridgedCloner - private let context: Context - private let cloningIntoFunction: Bool - - init(cloneTo global: GlobalVariable, _ context: Context) { - self.bridged = BridgedCloner(global.bridged, context._bridged) - self.context = context - self.cloningIntoFunction = false - } - - init(cloneBefore inst: Instruction, _ context: Context) { - self.bridged = BridgedCloner(inst.bridged, context._bridged) - self.context = context - self.cloningIntoFunction = true - } - - mutating func deinitialize() { - bridged.destroy(context._bridged) - } - - /// Transitively clones `value` including its defining instruction's operands. - mutating func clone(_ value: Value) -> Value { - - if isCloned(value: value) { - return getClonedValue(of: value) - } - - if let beginAccess = value as? BeginAccessInst { - // Skip access instructions, which might be generated for UnsafePointer globals which point to other globals. - let clonedOperand = clone(beginAccess.address) - bridged.recordFoldedValue(beginAccess.bridged, clonedOperand.bridged) - return clonedOperand - } - - let inst = value.definingInstruction! - assert(!(inst is ScopedInstruction), "global init value must not contain a scoped instruction") - - for op in inst.operands { - _ = clone(op.value) - } - - bridged.clone(inst.bridged) - let clonedValue = getClonedValue(of: value) - if cloningIntoFunction { - context.notifyInstructionChanged(clonedValue.definingInstruction!) - } - return clonedValue - } - - mutating func getClonedValue(of value: Value) -> Value { - bridged.getClonedValue(value.bridged).value - } - - func isCloned(value: Value) -> Bool { - bridged.isValueCloned(value.bridged) - } -} diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index 7ecf2ac06d2fb..deb7b1d44638a 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -51,7 +51,7 @@ class SILLoop; class BasicBlockSet; class NodeSet; class OperandSet; -class ClonerWithFixedLocation; +class BridgedClonerImpl; class SwiftPassInvocation; class FixedSizeSlabPayload; class FixedSizeSlab; @@ -220,26 +220,20 @@ struct BridgedOperandSet { }; struct BridgedCloner { - swift::ClonerWithFixedLocation * _Nonnull cloner; + swift::BridgedClonerImpl * _Nonnull cloner; BridgedCloner(BridgedGlobalVar var, BridgedPassContext context); BridgedCloner(BridgedInstruction inst, BridgedPassContext context); + BridgedCloner(BridgedFunction emptyFunction, BridgedPassContext context); void destroy(BridgedPassContext context); - SWIFT_IMPORT_UNSAFE BridgedValue getClonedValue(BridgedValue v); - bool isValueCloned(BridgedValue v) const; - void clone(BridgedInstruction inst); - void recordFoldedValue(BridgedValue origValue, BridgedValue mappedValue); -}; - -struct BridgedSpecializationCloner { - swift::SpecializationCloner * _Nonnull cloner; - - SWIFT_IMPORT_UNSAFE BridgedSpecializationCloner(BridgedFunction emptySpecializedFunction); SWIFT_IMPORT_UNSAFE BridgedFunction getCloned() const; SWIFT_IMPORT_UNSAFE BridgedBasicBlock getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const; void cloneFunctionBody(BridgedFunction originalFunction, BridgedBasicBlock clonedEntryBlock, BridgedValueArray clonedEntryBlockArgs) const; void cloneFunctionBody(BridgedFunction originalFunction) const; + SWIFT_IMPORT_UNSAFE BridgedValue getClonedValue(BridgedValue v); + bool isValueCloned(BridgedValue v) const; + BridgedInstruction clone(BridgedInstruction inst); }; struct BridgedPassContext { @@ -312,8 +306,7 @@ struct BridgedPassContext { BridgedOwnedString mangleOutlinedVariable(BridgedFunction function) const; BridgedOwnedString mangleAsyncRemoved(BridgedFunction function) const; BridgedOwnedString mangleWithDeadArgs(BridgedArrayRef bridgedDeadArgIndices, BridgedFunction function) const; - BridgedOwnedString mangleWithClosureArgs(BridgedValueArray closureArgs, - BridgedArrayRef closureArgIndices, + BridgedOwnedString mangleWithClosureArgs(BridgedArrayRef closureArgIndices, BridgedFunction applySiteCallee) const; BridgedOwnedString mangleWithBoxToStackPromotedArgs(BridgedArrayRef bridgedPromotedArgIndices, BridgedFunction bridgedOriginalFunction) const; diff --git a/lib/SILOptimizer/LoopTransforms/LICM.cpp b/lib/SILOptimizer/LoopTransforms/LICM.cpp index 567990e355560..ceba90e5cf9cd 100644 --- a/lib/SILOptimizer/LoopTransforms/LICM.cpp +++ b/lib/SILOptimizer/LoopTransforms/LICM.cpp @@ -1486,6 +1486,7 @@ hoistLoadsAndStores(AccessPath accessPath, SILLoop *loop) { continue; } auto loadWithAccess = isLoadWithinAccess(I, accessPath); + if (!loadWithAccess) { continue; } diff --git a/lib/SILOptimizer/Utils/OptimizerBridging.cpp b/lib/SILOptimizer/Utils/OptimizerBridging.cpp index 9e12ed2284192..8d37156304f27 100644 --- a/lib/SILOptimizer/Utils/OptimizerBridging.cpp +++ b/lib/SILOptimizer/Utils/OptimizerBridging.cpp @@ -308,32 +308,31 @@ BridgedOwnedString BridgedPassContext::mangleWithDeadArgs(BridgedArrayRef bridge } BridgedOwnedString BridgedPassContext::mangleWithClosureArgs( - BridgedValueArray bridgedClosureArgs, - BridgedArrayRef bridgedClosureArgIndices, - BridgedFunction applySiteCallee + BridgedArrayRef bridgedClosureArgs, BridgedFunction applySiteCallee ) const { + + struct ClosureArgElement { + SwiftInt argIdx; + BridgeValueExistential argValue; + }; + auto pass = Demangle::SpecializationPass::ClosureSpecializer; auto serializedKind = applySiteCallee.getFunction()->getSerializedKind(); Mangle::FunctionSignatureSpecializationMangler mangler(applySiteCallee.getFunction()->getASTContext(), pass, serializedKind, applySiteCallee.getFunction()); - llvm::SmallVector closureArgsStorage; - auto closureArgs = bridgedClosureArgs.getValues(closureArgsStorage); - auto closureArgIndices = bridgedClosureArgIndices.unbridged(); + auto closureArgs = bridgedClosureArgs.unbridged(); - assert(closureArgs.size() == closureArgIndices.size() && - "Number of closures arguments and number of closure indices do not match!"); - - for (size_t i = 0; i < closureArgs.size(); i++) { - auto closureArg = closureArgs[i]; - auto closureArgIndex = closureArgIndices[i]; + for (ClosureArgElement argElmt : closureArgs) { + auto closureArg = argElmt.argValue.value.getSILValue(); + auto closureArgIndex = argElmt.argIdx; if (auto *PAI = dyn_cast(closureArg)) { mangler.setArgumentClosureProp(closureArgIndex, const_cast(PAI)); } else { auto *TTTFI = cast(closureArg); - mangler.setArgumentClosureProp(closureArgIndex, + mangler.setArgumentClosureProp(closureArgIndex, const_cast(TTTFI)); } } @@ -577,52 +576,89 @@ BridgedCalleeAnalysis::CalleeList BridgedCalleeAnalysis::getDestructors(BridgedT // it in OptimizerBridging.h. namespace swift { -class ClonerWithFixedLocation : public SILCloner { - friend class SILInstructionVisitor; - friend class SILCloner; +class BridgedClonerImpl : public SILCloner { + friend class SILInstructionVisitor; + friend class SILCloner; - SILDebugLocation insertLoc; + bool hasFixedLocation; + union { + SILDebugLocation fixedLocation; + ScopeCloner scopeCloner; + }; -public: - ClonerWithFixedLocation(SILGlobalVariable *gVar) - : SILCloner(gVar), - insertLoc(ArtificialUnreachableLocation(), nullptr) {} + SILInstruction *result = nullptr; - ClonerWithFixedLocation(SILInstruction *insertionPoint) - : SILCloner(*insertionPoint->getFunction()), - insertLoc(insertionPoint->getDebugLocation()) { +public: + BridgedClonerImpl(SILGlobalVariable *gVar) + : SILCloner(gVar), + hasFixedLocation(true), + fixedLocation(ArtificialUnreachableLocation(), nullptr) {} + + BridgedClonerImpl(SILInstruction *insertionPoint) + : SILCloner(*insertionPoint->getFunction()), + hasFixedLocation(true), + fixedLocation(insertionPoint->getDebugLocation()) { Builder.setInsertionPoint(insertionPoint); } + BridgedClonerImpl(SILFunction &emptyFunction) + : SILCloner(emptyFunction), + hasFixedLocation(false), + scopeCloner(ScopeCloner(emptyFunction)) {} + + ~BridgedClonerImpl() { + if (hasFixedLocation) { + fixedLocation.~SILDebugLocation(); + } else { + scopeCloner.~ScopeCloner(); + } + } + SILValue getClonedValue(SILValue v) { return getMappedValue(v); } - void cloneInst(SILInstruction *inst) { + SILInstruction *cloneInst(SILInstruction *inst) { + result = nullptr; visit(inst); + ASSERT(result && "instruction not cloned"); + return result; } -protected: - SILLocation remapLocation(SILLocation loc) { - return insertLoc.getLocation(); + if (hasFixedLocation) + return fixedLocation.getLocation(); + return loc; } const SILDebugScope *remapScope(const SILDebugScope *DS) { - return insertLoc.getScope(); + if (hasFixedLocation) + return fixedLocation.getScope(); + return scopeCloner.getOrCreateClonedScope(DS); + } + + void postProcess(SILInstruction *Orig, SILInstruction *Cloned) { + result = Cloned; + SILCloner::postProcess(Orig, Cloned); } + }; } // namespace swift BridgedCloner::BridgedCloner(BridgedGlobalVar var, BridgedPassContext context) - : cloner(new ClonerWithFixedLocation(var.getGlobal())) { + : cloner(new BridgedClonerImpl(var.getGlobal())) { context.invocation->notifyNewCloner(); } BridgedCloner::BridgedCloner(BridgedInstruction inst, BridgedPassContext context) - : cloner(new ClonerWithFixedLocation(inst.unbridged())) { + : cloner(new BridgedClonerImpl(inst.unbridged())) { + context.invocation->notifyNewCloner(); +} + +BridgedCloner::BridgedCloner(BridgedFunction emptyFunction, BridgedPassContext context) + : cloner(new BridgedClonerImpl(*emptyFunction.getFunction())) { context.invocation->notifyNewCloner(); } @@ -632,6 +668,10 @@ void BridgedCloner::destroy(BridgedPassContext context) { context.invocation->notifyClonerDestroyed(); } +BridgedFunction BridgedCloner::getCloned() const { + return { &cloner->getBuilder().getFunction() }; +} + BridgedValue BridgedCloner::getClonedValue(BridgedValue v) { return {cloner->getClonedValue(v.getSILValue())}; } @@ -640,51 +680,32 @@ bool BridgedCloner::isValueCloned(BridgedValue v) const { return cloner->isValueCloned(v.getSILValue()); } -void BridgedCloner::clone(BridgedInstruction inst) { - cloner->cloneInst(inst.unbridged()); -} - -void BridgedCloner::recordFoldedValue(BridgedValue origValue, BridgedValue mappedValue) { - cloner->recordFoldedValue(origValue.getSILValue(), mappedValue.getSILValue()); -} - -namespace swift { - class SpecializationCloner: public SILClonerWithScopes { - friend class SILInstructionVisitor; - friend class SILCloner; - public: - using SuperTy = SILClonerWithScopes; - SpecializationCloner(SILFunction &emptySpecializedFunction): SuperTy(emptySpecializedFunction) {} - }; -} // namespace swift - -BridgedSpecializationCloner::BridgedSpecializationCloner(BridgedFunction emptySpecializedFunction): - cloner(new SpecializationCloner(*emptySpecializedFunction.getFunction())) {} - -BridgedFunction BridgedSpecializationCloner::getCloned() const { - return { &cloner->getBuilder().getFunction() }; +BridgedInstruction BridgedCloner::clone(BridgedInstruction inst) { + return {cloner->cloneInst(inst.unbridged())->asSILNode()}; } -BridgedBasicBlock BridgedSpecializationCloner::getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const { +BridgedBasicBlock BridgedCloner::getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const { return { cloner->getOpBasicBlock(originalBasicBlock.unbridged()) }; } -void BridgedSpecializationCloner::cloneFunctionBody(BridgedFunction originalFunction, BridgedBasicBlock clonedEntryBlock, BridgedValueArray clonedEntryBlockArgs) const { +void BridgedCloner::cloneFunctionBody(BridgedFunction originalFunction, + BridgedBasicBlock clonedEntryBlock, + BridgedValueArray clonedEntryBlockArgs) const { llvm::SmallVector clonedEntryBlockArgsStorage; auto clonedEntryBlockArgsArrayRef = clonedEntryBlockArgs.getValues(clonedEntryBlockArgsStorage); cloner->cloneFunctionBody(originalFunction.getFunction(), clonedEntryBlock.unbridged(), clonedEntryBlockArgsArrayRef); } -void BridgedSpecializationCloner::cloneFunctionBody(BridgedFunction originalFunction) const { +void BridgedCloner::cloneFunctionBody(BridgedFunction originalFunction) const { cloner->cloneFunction(originalFunction.getFunction()); } void BridgedBuilder::destroyCapturedArgs(BridgedInstruction partialApply) const { if (auto *pai = llvm::dyn_cast(partialApply.unbridged()); pai->isOnStack()) { auto b = unbridged(); - return swift::insertDestroyOfCapturedArguments(pai, b); + return swift::insertDestroyOfCapturedArguments(pai, b); } else { - assert(false && "`destroyCapturedArgs` must only be called on a `partial_apply` on stack!"); + assert(false && "`destroyCapturedArgs` must only be called on a `partial_apply` on stack!"); } } diff --git a/utils/vim/ftdetect/sil.vim b/utils/vim/ftdetect/sil.vim deleted file mode 100644 index 7e0f0e2934bc7..0000000000000 --- a/utils/vim/ftdetect/sil.vim +++ /dev/null @@ -1,9 +0,0 @@ -" This source file is part of the Swift.org open source project -" -" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -" Licensed under Apache License v2.0 with Runtime Library Exception -" -" See https://swift.org/LICENSE.txt for license information -" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors - -au BufNewFile,BufRead *.sil set ft=sil diff --git a/utils/vim/ftdetect/swift.vim b/utils/vim/ftdetect/swift.vim deleted file mode 100644 index 4812f57fe0edd..0000000000000 --- a/utils/vim/ftdetect/swift.vim +++ /dev/null @@ -1,9 +0,0 @@ -" This source file is part of the Swift.org open source project -" -" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -" Licensed under Apache License v2.0 with Runtime Library Exception -" -" See https://swift.org/LICENSE.txt for license information -" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors - -au BufNewFile,BufRead *.swift set ft=swift diff --git a/utils/vim/ftdetect/swiftgyb.vim b/utils/vim/ftdetect/swiftgyb.vim deleted file mode 100644 index 3dd91226f7194..0000000000000 --- a/utils/vim/ftdetect/swiftgyb.vim +++ /dev/null @@ -1,9 +0,0 @@ -" This source file is part of the Swift.org open source project -" -" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -" Licensed under Apache License v2.0 with Runtime Library Exception -" -" See https://swift.org/LICENSE.txt for license information -" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors - -au BufNewFile,BufRead *.swift.gyb set ft=swiftgyb diff --git a/utils/vim/ftplugin/swift.vim b/utils/vim/ftplugin/swift.vim deleted file mode 100644 index 55d9e47b6398a..0000000000000 --- a/utils/vim/ftplugin/swift.vim +++ /dev/null @@ -1,21 +0,0 @@ -" This source file is part of the Swift.org open source project -" -" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -" Licensed under Apache License v2.0 with Runtime Library Exception -" -" See https://swift.org/LICENSE.txt for license information -" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors - -" Only do this when not done yet for this buffer -if exists("b:did_ftplugin") - finish -endif - -let b:did_ftplugin = 1 -let b:undo_ftplugin = "setlocal comments< expandtab< tabstop< shiftwidth< smartindent<" - -setlocal comments=s1:/*,mb:*,ex:*/,:///,:// -setlocal expandtab -setlocal tabstop=2 -setlocal shiftwidth=2 -setlocal smartindent diff --git a/utils/vim/ftplugin/swiftgyb.vim b/utils/vim/ftplugin/swiftgyb.vim deleted file mode 100644 index 033b60c6729f4..0000000000000 --- a/utils/vim/ftplugin/swiftgyb.vim +++ /dev/null @@ -1,9 +0,0 @@ -" This source file is part of the Swift.org open source project -" -" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -" Licensed under Apache License v2.0 with Runtime Library Exception -" -" See https://swift.org/LICENSE.txt for license information -" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors - -runtime! ftplugin/swift.vim diff --git a/utils/vim/syntax/sil.vim b/utils/vim/syntax/sil.vim deleted file mode 100644 index b043cd3d4ce77..0000000000000 --- a/utils/vim/syntax/sil.vim +++ /dev/null @@ -1,171 +0,0 @@ -" This source file is part of the Swift.org open source project -" -" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -" Licensed under Apache License v2.0 with Runtime Library Exception -" -" See https://swift.org/LICENSE.txt for license information -" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -" -" Vim syntax file -" Language: sil - -if exists("b:current_syntax") - finish -endif - -syn keyword silStage skipwhite nextgroup=silStages - \ sil_stage -syn keyword silStages - \ canonical - \ raw - -syn match silIdentifier skipwhite - \ /@\<[A-Za-z_0-9]\+\>/ - -syn match silConvention skipwhite - \ /$\?@convention/ -syn region silConvention contained contains=silConventions - \ start="@convention(" end=")" -syn keyword silConventions - \ block - \ c - \ method - \ objc_method - \ sil_differentiability_witness - \ thick - \ thin - \ witness_method - -syn match silFunctionType skipwhite - \ /@\(\\|\\|\\|\\|\\|\\|\\|\\|\\|\\|\\|\\|\\)/ -syn match silMetatypeType skipwhite - \ /@\(\\|\\|\\)/ - -" TODO: handle [tail_elems sil-type * sil-operand] -syn region silAttribute contains=silAttributes - \ start="\[" end="\]" -syn keyword silAttributes contained containedin=silAttribute - \ abort - \ deinit - \ delegatingself - \ derivedself - \ derivedselfonly - \ dynamic - \ exact - \ init - \ modify - \ mutating - \ objc - \ open - \ read - \ rootself - \ stack - \ static - \ strict - \ unknown - \ unsafe - \ var - -syn keyword swiftImport import skipwhite nextgroup=swiftImportModule -syn match swiftImportModule /\<[A-Za-z_][A-Za-z_0-9]*\>/ contained nextgroup=swiftImportComponent -syn match swiftImportComponent /\.\<[A-Za-z_][A-Za-z_0-9]*\>/ contained nextgroup=swiftImportComponent - -syn region swiftComment start="/\*" end="\*/" contains=swiftComment,swiftTodo -syn region swiftLineComment start="//" end="$" contains=swiftTodo - -syn match swiftLineComment /^#!.*/ -syn match swiftTypeName /\<[A-Z][a-zA-Z_0-9]*\>/ -syn match swiftDecimal /\<[-]\?[0-9]\+\>/ -syn match swiftDecimal /\<[-+]\?[0-9]\+\>/ - -syn match swiftTypeName /\$\*\<\?[A-Z][a-zA-Z0-9_]*\>/ -syn match swiftVarName /%\<[A-z[a-z_0-9]\+\(#[0-9]\+\)\?\>/ - -syn keyword swiftKeyword break case continue default do else for if in static switch repeat return where while skipwhite - -syn keyword swiftKeyword sil internal thunk skipwhite -syn keyword swiftKeyword public hidden private shared public_external hidden_external skipwhite -syn keyword swiftKeyword getter setter allocator initializer enumelt destroyer globalaccessor objc skipwhite -syn keyword swiftKeyword alloc_global alloc_stack alloc_ref alloc_ref_dynamic alloc_box alloc_existential_box dealloc_stack dealloc_stack_ref dealloc_box dealloc_existential_box dealloc_ref dealloc_partial_ref skipwhite -syn keyword swiftKeyword debug_value debug_value_addr skipwhite -syn keyword swiftKeyword load load_unowned store assign mark_uninitialized mark_function_escape copy_addr destroy_addr index_addr index_raw_pointer bind_memory to skipwhite -syn keyword swiftKeyword strong_retain strong_release strong_retain_unowned ref_to_unowned unowned_to_ref unowned_retain unowned_release load_weak store_unowned store_weak fix_lifetime autorelease_value set_deallocating is_unique destroy_not_escaped_closure skipwhite -syn keyword swiftKeyword function_ref integer_literal float_literal string_literal global_addr skipwhite -syn keyword swiftKeyword class_method super_method witness_method objc_method objc_super_method skipwhite -syn keyword swiftKeyword partial_apply builtin skipwhite -syn keyword swiftApplyKeyword apply try_apply skipwhite -syn keyword swiftKeyword metatype value_metatype existential_metatype skipwhite -syn keyword swiftKeyword retain_value release_value retain_value_addr release_value_addr tuple tuple_extract tuple_element_addr struct struct_extract struct_element_addr ref_element_addr skipwhite -syn keyword swiftKeyword init_enum_data_addr unchecked_enum_data unchecked_take_enum_data_addr inject_enum_addr skipwhite -syn keyword swiftKeyword init_existential_addr init_existential_value init_existential_metatype deinit_existential_addr deinit_existential_value open_existential_addr open_existential_box open_existential_box_value open_existential_metatype init_existential_ref open_existential_ref open_existential_value skipwhite -syn keyword swiftKeyword upcast address_to_pointer pointer_to_address unchecked_addr_cast unchecked_ref_cast unchecked_ref_cast_addr ref_to_raw_pointer ref_to_bridge_object ref_to_unmanaged unmanaged_to_ref raw_pointer_to_ref skipwhite -syn keyword swiftKeyword convert_function thick_to_objc_metatype objc_to_thick_metatype thin_to_thick_function unchecked_ref_bit_cast unchecked_trivial_bit_cast bridge_object_to_ref bridge_object_to_word unchecked_bitwise_cast skipwhite -syn keyword swiftKeyword objc_existential_metatype_to_object objc_metatype_to_object objc_protocol skipwhite -syn keyword swiftKeyword unconditional_checked_cast unconditional_checked_cast_addr unconditional_checked_cast_value skipwhite -syn keyword swiftKeyword cond_fail skipwhite -syn keyword swiftKeyword unreachable return throw br cond_br switch_value select_enum select_enum_addr switch_enum switch_enum_addr dynamic_method_br checked_cast_br checked_cast_value_br checked_cast_addr_br skipwhite -syn keyword swiftKeyword project_box project_existential_box project_block_storage init_block_storage_header copy_block mark_dependence skipwhite - -syn keyword swiftTypeDefinition class extension protocol struct typealias enum skipwhite nextgroup=swiftTypeName -syn region swiftTypeAttributes start="\[" end="\]" skipwhite contained nextgroup=swiftTypeName -syn match swiftTypeName /\<[A-Za-z_][A-Za-z_0-9\.]*\>/ contained nextgroup=swiftTypeParameters - -syn region swiftTypeParameters start="<" end=">" skipwhite contained - -syn keyword swiftFuncDefinition func skipwhite nextgroup=swiftFuncAttributes,swiftFuncName,swiftOperator -syn region swiftFuncAttributes start="\[" end="\]" skipwhite contained nextgroup=swiftFuncName,swiftOperator -syn match swiftFuncName /\<[A-Za-z_][A-Za-z_0-9]*\>/ skipwhite contained nextgroup=swiftTypeParameters -syn keyword swiftFuncKeyword subscript init destructor nextgroup=swiftTypeParameters - -syn keyword swiftVarDefinition var skipwhite nextgroup=swiftVarName -syn keyword swiftVarDefinition let skipwhite nextgroup=swiftVarName -syn match swiftVarName /\<[A-Za-z_][A-Za-z_0-9]*\>/ skipwhite contained - -syn keyword swiftDefinitionModifier static - -syn match swiftImplicitVarName /\$\<[A-Za-z_0-9]\+\>/ - -hi def link swiftImport Include -hi def link swiftImportModule Title -hi def link swiftImportComponent Identifier -hi def link swiftApplyKeyword Statement -hi def link swiftKeyword Statement -hi def link swiftTypeDefinition Define -hi def link swiftTypeName Type -hi def link swiftTypeParameters Special -hi def link swiftTypeAttributes PreProc -hi def link swiftFuncDefinition Define -hi def link swiftDefinitionModifier Define -hi def link swiftFuncName Function -hi def link swiftFuncAttributes PreProc -hi def link swiftFuncKeyword Function -hi def link swiftVarDefinition Define -hi def link swiftVarName Identifier -hi def link swiftImplicitVarName Identifier -hi def link swiftIdentifierKeyword Identifier -hi def link swiftTypeDeclaration Delimiter -hi def link swiftBoolean Boolean -hi def link swiftString String -hi def link swiftInterpolation Special -hi def link swiftComment Comment -hi def link swiftLineComment Comment -hi def link swiftDecimal Number -hi def link swiftHex Number -hi def link swiftOct Number -hi def link swiftBin Number -hi def link swiftOperator Function -hi def link swiftChar Character -hi def link swiftLabel Label -hi def link swiftNew Operator - -hi def link silStage Special -hi def link silStages Type -hi def link silConvention Special -hi def link silConventionParameter Special -hi def link silConventions Type -hi def link silIdentifier Identifier -hi def link silFunctionType Special -hi def link silMetatypeType Special -hi def link silAttribute PreProc - -let b:current_syntax = "sil" diff --git a/utils/vim/syntax/swift.vim b/utils/vim/syntax/swift.vim deleted file mode 100644 index baab940550fcd..0000000000000 --- a/utils/vim/syntax/swift.vim +++ /dev/null @@ -1,285 +0,0 @@ -" This source file is part of the Swift.org open source project -" -" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -" Licensed under Apache License v2.0 with Runtime Library Exception -" -" See https://swift.org/LICENSE.txt for license information -" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -" -" Vim syntax file -" Language: swift -" Maintainer: Joe Groff -" Last Change: 2018 Jan 21 - -if exists("b:current_syntax") - finish -endif - -syn keyword swiftKeyword - \ await - \ break - \ case - \ catch - \ continue - \ default - \ defer - \ do - \ else - \ fallthrough - \ for - \ guard - \ if - \ in - \ repeat - \ return - \ switch - \ throw - \ try - \ where - \ while -syn match swiftMultiwordKeyword - \ "indirect case" - -syn keyword swiftCoreTypes - \ Any - \ AnyObject - -syn keyword swiftImport skipwhite skipempty nextgroup=swiftImportModule - \ import - -syn keyword swiftDefinitionModifier - \ async - \ convenience - \ dynamic - \ fileprivate - \ final - \ internal - \ lazy - \ nonmutating - \ open - \ override - \ prefix - \ private - \ public - \ reasync - \ required - \ rethrows - \ static - \ throws - \ weak - -syn keyword swiftInOutKeyword skipwhite skipempty nextgroup=swiftTypeName - \ inout - -syn keyword swiftIdentifierKeyword - \ Self - \ metatype - \ self - \ super - -syn keyword swiftFuncKeywordGeneral skipwhite skipempty nextgroup=swiftTypeParameters - \ init - -syn keyword swiftFuncKeyword - \ deinit - \ subscript - -syn keyword swiftScope - \ autoreleasepool - -syn keyword swiftMutating skipwhite skipempty nextgroup=swiftFuncDefinition - \ mutating -syn keyword swiftFuncDefinition skipwhite skipempty nextgroup=swiftTypeName,swiftOperator - \ func - -syn keyword swiftTypeDefinition skipwhite skipempty nextgroup=swiftTypeName - \ class - \ enum - \ extension - \ operator - \ precedencegroup - \ protocol - \ struct - -syn keyword swiftTypeAliasDefinition skipwhite skipempty nextgroup=swiftTypeAliasName - \ associatedtype - \ typealias - -syn match swiftMultiwordTypeDefinition skipwhite skipempty nextgroup=swiftTypeName - \ "indirect enum" - -syn keyword swiftVarDefinition skipwhite skipempty nextgroup=swiftVarName - \ let - \ var - -syn keyword swiftLabel - \ get - \ set - \ didSet - \ willSet - -syn keyword swiftBoolean - \ false - \ true - -syn keyword swiftNil - \ nil - -syn match swiftImportModule contained nextgroup=swiftImportComponent - \ /\<[A-Za-z_][A-Za-z_0-9]*\>/ -syn match swiftImportComponent contained nextgroup=swiftImportComponent - \ /\.\<[A-Za-z_][A-Za-z_0-9]*\>/ - -syn match swiftTypeAliasName contained skipwhite skipempty nextgroup=swiftTypeAliasValue - \ /\<[A-Za-z_][A-Za-z_0-9]*\>/ -syn match swiftTypeName contained skipwhite skipempty nextgroup=swiftTypeParameters - \ /\<[A-Za-z_][A-Za-z_0-9\.]*\>/ -syn match swiftVarName contained skipwhite skipempty nextgroup=swiftTypeDeclaration - \ /\<[A-Za-z_][A-Za-z_0-9]*\>/ -syn match swiftImplicitVarName - \ /\$\<[A-Za-z_0-9]\+\>/ - -" TypeName[Optionality]? -syn match swiftType contained skipwhite skipempty nextgroup=swiftTypeParameters - \ /\<[A-Za-z_][A-Za-z_0-9\.]*\>[!?]\?/ -" [Type:Type] (dictionary) or [Type] (array) -syn region swiftType contained contains=swiftTypePair,swiftType - \ matchgroup=Delimiter start=/\[/ end=/\]/ -syn match swiftTypePair contained skipwhite skipempty nextgroup=swiftTypeParameters,swiftTypeDeclaration - \ /\<[A-Za-z_][A-Za-z_0-9\.]*\>[!?]\?/ -" (Type[, Type]) (tuple) -" FIXME: we should be able to use skip="," and drop swiftParamDelim -syn region swiftType contained contains=swiftType,swiftParamDelim - \ matchgroup=Delimiter start="[^@]\?(" end=")" matchgroup=NONE skip="," -syn match swiftParamDelim contained - \ /,/ -" (generics) -syn region swiftTypeParameters contained contains=swiftVarName,swiftConstraint - \ matchgroup=Delimiter start="<" end=">" matchgroup=NONE skip="," -syn keyword swiftConstraint contained - \ where - -syn match swiftTypeAliasValue skipwhite skipempty nextgroup=swiftType - \ /=/ -syn match swiftTypeDeclaration skipwhite skipempty nextgroup=swiftType,swiftInOutKeyword - \ /:/ -syn match swiftTypeDeclaration skipwhite skipempty nextgroup=swiftType - \ /->/ - -syn match swiftKeyword - \ /\/ -syn region swiftCaseLabelRegion - \ matchgroup=swiftKeyword start=/\/ matchgroup=Delimiter end=/:/ oneline contains=TOP -syn region swiftDefaultLabelRegion - \ matchgroup=swiftKeyword start=/\/ matchgroup=Delimiter end=/:/ oneline - -syn region swiftParenthesisRegion contains=TOP - \ matchgroup=NONE start=/(/ end=/)/ - -syn region swiftString contains=swiftInterpolationRegion - \ start=/"/ skip=/\\\\\|\\"/ end=/"/ -syn region swiftInterpolationRegion contained contains=TOP - \ matchgroup=swiftInterpolation start=/\\(/ end=/)/ -syn region swiftComment contains=swiftComment,swiftTodo - \ start="/\*" end="\*/" -syn region swiftLineComment contains=swiftTodo - \ start="//" end="$" - -syn match swiftDecimal - \ /[+\-]\?\<\([0-9][0-9_]*\)\([.][0-9_]*\)\?\([eE][+\-]\?[0-9][0-9_]*\)\?\>/ -syn match swiftHex - \ /[+\-]\?\<0x[0-9A-Fa-f][0-9A-Fa-f_]*\(\([.][0-9A-Fa-f_]*\)\?[pP][+\-]\?[0-9][0-9_]*\)\?\>/ -syn match swiftOct - \ /[+\-]\?\<0o[0-7][0-7_]*\>/ -syn match swiftBin - \ /[+\-]\?\<0b[01][01_]*\>/ - -syn match swiftOperator skipwhite skipempty nextgroup=swiftTypeParameters - \ "\.\@!&|^~]\@!&|^~]*\|*/\@![/=\-+*%<>!&|^~]*\|->\@![/=\-+*%<>!&|^~]*\|[=+%<>!&|^~][/=\-+*%<>!&|^~]*\)" -syn match swiftOperator skipwhite skipempty nextgroup=swiftTypeParameters - \ "\.\.[<.]" - -syn match swiftChar - \ /'\([^'\\]\|\\\(["'tnr0\\]\|x[0-9a-fA-F]\{2}\|u[0-9a-fA-F]\{4}\|U[0-9a-fA-F]\{8}\)\)'/ - -syn match swiftTupleIndexNumber contains=swiftDecimal - \ /\.[0-9]\+/ -syn match swiftDecimal contained - \ /[0-9]\+/ - -" This is a superset of the Preproc macros below, so it must come FIRST -syn match swiftFreestandingMacro - \ /#\<[A-Za-z_][A-Za-z_0-9]*\>/ -syn match swiftPreproc - \ /#\(\\|\\|\\|\\|\\)/ -syn match swiftPreproc - \ /^\s*#\(\\|\\|\\|\\|\\|\\)/ -syn region swiftPreprocFalse - \ start="^\s*#\\s\+\" end="^\s*#\(\\|\\|\\)" - -syn match swiftAttribute - \ /@\<\w\+\>/ skipwhite skipempty nextgroup=swiftType,swiftTypeDefinition - -syn keyword swiftTodo MARK TODO FIXME contained - -syn match swiftCastOp skipwhite skipempty nextgroup=swiftType,swiftCoreTypes - \ "\" -syn match swiftCastOp skipwhite skipempty nextgroup=swiftType,swiftCoreTypes - \ "\[!?]\?" - -syn match swiftNilOps - \ "??" - -syn region swiftReservedIdentifier oneline - \ start=/`/ end=/`/ - -hi def link swiftImport Include -hi def link swiftImportModule Title -hi def link swiftImportComponent Identifier -hi def link swiftKeyword Statement -hi def link swiftCoreTypes Type -hi def link swiftMultiwordKeyword Statement -hi def link swiftTypeDefinition Define -hi def link swiftMultiwordTypeDefinition Define -hi def link swiftType Type -hi def link swiftTypePair Type -hi def link swiftTypeAliasName Identifier -hi def link swiftTypeName Function -hi def link swiftConstraint Special -hi def link swiftFuncDefinition Define -hi def link swiftDefinitionModifier Operator -hi def link swiftInOutKeyword Define -hi def link swiftFuncKeyword Function -hi def link swiftFuncKeywordGeneral Function -hi def link swiftTypeAliasDefinition Define -hi def link swiftVarDefinition Define -hi def link swiftVarName Identifier -hi def link swiftImplicitVarName Identifier -hi def link swiftIdentifierKeyword Identifier -hi def link swiftTypeAliasValue Delimiter -hi def link swiftTypeDeclaration Delimiter -hi def link swiftTypeParameters Delimiter -hi def link swiftBoolean Boolean -hi def link swiftString String -hi def link swiftInterpolation Special -hi def link swiftComment Comment -hi def link swiftLineComment Comment -hi def link swiftDecimal Number -hi def link swiftHex Number -hi def link swiftOct Number -hi def link swiftBin Number -hi def link swiftOperator Function -hi def link swiftChar Character -hi def link swiftLabel Operator -hi def link swiftMutating Statement -hi def link swiftPreproc PreCondit -hi def link swiftPreprocFalse Comment -hi def link swiftFreestandingMacro Macro -hi def link swiftAttribute Type -hi def link swiftTodo Todo -hi def link swiftNil Constant -hi def link swiftCastOp Operator -hi def link swiftNilOps Operator -hi def link swiftScope PreProc - -let b:current_syntax = "swift" diff --git a/utils/vim/syntax/swiftgyb.vim b/utils/vim/syntax/swiftgyb.vim deleted file mode 100644 index 300da06dd3726..0000000000000 --- a/utils/vim/syntax/swiftgyb.vim +++ /dev/null @@ -1,22 +0,0 @@ -" This source file is part of the Swift.org open source project -" -" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors -" Licensed under Apache License v2.0 with Runtime Library Exception -" -" See https://swift.org/LICENSE.txt for license information -" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -" -" Vim syntax file -" Language: gyb on swift - -runtime! syntax/swift.vim -unlet b:current_syntax - -syn include @Python syntax/python.vim -syn region pythonCode matchgroup=gybPythonCode start=+^ *%+ end=+$+ contains=@Python keepend -syn region pythonCode matchgroup=gybPythonCode start=+%{+ end=+}%+ contains=@Python keepend -syn match gybPythonCode /\${[^}]*}/ -hi def link gybPythonCode CursorLineNr - -let b:current_syntax = "swiftgyb" - From d9f59c22c426cbf1adcd1f93293d78ac4b11b6bf Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Thu, 24 Jul 2025 16:29:25 +0100 Subject: [PATCH 11/44] Add . Minor fixes. --- .../LoopInvariantCodeMotion.swift | 42 +++++----- .../Sources/Optimizer/Utilities/Cloner.swift | 22 ++++- test/SILOptimizer/licm.sil | 84 ------------------- 3 files changed, 39 insertions(+), 109 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index a15ccdb7245f2..a8f23305a8713 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -21,6 +21,7 @@ let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion context: context ) } + context.removeTriviallyDeadInstructionsIgnoringDebugUses(in: function) } struct DiscoveredMovableInstructions { @@ -424,19 +425,16 @@ func hoistAllLoadsAndStores( context: context ) } - // TODO: Should we clear the data structures or have some other way of not trying to re-optimize same instructions? guard !movableInstructions.toDelete.isEmpty else { return false } for inst in movableInstructions.toDelete { - // TODO: Is this equivalent with the C++ code? Should we just bridge the function? if inst.isTriviallyDead { context.erase(instruction: inst) } } - // TODO: Same us further up. Should we clean the data structure? return true } @@ -537,15 +535,19 @@ func hoistLoadsAndStores( isLoadWithAccessPath(loadInst, thatOverlapsAccess: accessPath) else { continue } + + let rootVal = currentVal ?? ssaUpdater.getValue(inMiddleOf: block) - let projectedValue = projectLoadValue( + guard let projectedValue = projectLoadValue( addr: loadInst.operand.value, accessPath: loadInst.operand.value.accessPath, - rootVal: currentVal ?? ssaUpdater.getValue(inMiddleOf: block), + rootVal: rootVal, rootAccessPath: accessPath, beforeInst: loadInst, context: context - ) + ) else { + continue + } loadInst.replace(with: projectedValue, context) } @@ -579,29 +581,26 @@ func projectLoadValue( rootAccessPath: AccessPath, beforeInst: Instruction, context: FunctionPassContext -) -> Value { +) -> Value? { guard accessPath != rootAccessPath else { return rootVal } guard let projectionPath = rootAccessPath.getProjection(to: accessPath) else { - fatalError() + return nil } - let (kind, index, path) = projectionPath.pop() + let builder = Builder(before: beforeInst, context) + let (kind, index, _) = projectionPath.pop() - // MARK: Finish this function. -// if let structElementAddrInst = addr as? StructElementAddrInst { -// let builder = Builder(before: beforeInst, context) -// return builder.createStructExtract(struct: structElementAddrInst.operand.value, fieldIndex: index) -// } else if let tupleElementAddrInst = addr as? TupleElementAddrInst { -// let builder = Builder(before: beforeInst, context) -// return builder.createStructExtract(struct: tupleElementAddrInst.operand.value, fieldIndex: index) -// } - - // MARK: Implement with new access path. Wait for example from Erik. - - return rootVal + switch kind { + case .structField: + return builder.createStructExtract(struct: rootVal, fieldIndex: index) + case .tupleField: + return builder.createTupleExtract(tuple: rootVal, elementIndex: index) + default: + return nil + } } func storesCommonlyDominateLoopExits( @@ -1279,6 +1278,7 @@ private func isCoveredByScope( extension AccessPath { fileprivate func isLoopInvariant(loop: Loop) -> Bool { + // %7 = pointer_to_address %1 : $Builtin.RawPointer to $*Index // user: %8 switch base { case .box(let inst as Instruction), .class(let inst as Instruction), .index(let inst as Instruction), diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift index 72910156a22d6..40e7ff3ce92e1 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift @@ -99,19 +99,33 @@ struct Cloner { // end_borrows or fix mark_dependence operands. case is BeginBorrowInst, is MarkDependenceInst: return nil case let singleValueInstruction as SingleValueInstruction: +// guard shouldClone(singleValueInstruction) else { return nil } // TODO: Double check whether correct guard let sourceOperand = singleValueInstruction.operands.first else { return nil } - return cloneProjection(projectAddr: singleValueInstruction, sourceOperand: sourceOperand) + return cloneProjection(projectAddr: singleValueInstruction, sourceOperand: sourceOperand, checkBase: checkBase) default: return nil } } + private func shouldClone(_ singleValueInstruction: SingleValueInstruction) -> Bool { + switch singleValueInstruction { + case is StructElementAddrInst, is TupleElementAddrInst, is IndexAddrInst, is TailAddrInst, is InitEnumDataAddrInst, is OpenExistentialAddrInst, is UncheckedTakeEnumDataAddrInst, is ProjectBoxInst, is ProjectBlockStorageInst, is MoveOnlyWrapperToCopyableAddrInst, is CopyableToMoveOnlyWrapperAddrInst, is MoveOnlyWrapperToCopyableBoxInst, is UncheckedAddrCastInst, is AddressToPointerInst, is PointerToAddressInst, is MarkUninitializedInst, is MarkUnresolvedReferenceBindingInst, is DropDeinitInst, is MarkUnresolvedReferenceBindingInst, is MarkDependenceInst, is CopyValueInst, is BeginBorrowInst, is StoreBorrowInst: return true + default: return false + } + } + private mutating func cloneProjection( projectAddr: SingleValueInstruction, - sourceOperand: Operand - ) -> Value { - let projectedSource = cloneRecursively(value: sourceOperand.value) + sourceOperand: Operand, + checkBase: (Value) -> Bool + ) -> Value? { + guard let projectedSource = cloneUseDefChain( + addr: sourceOperand.value, + checkBase: checkBase + ) else { + return nil + } let clone = clone(instruction: projectAddr) clone.setOperand(at: sourceOperand.index, to: projectedSource, context) diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index e149624646a1d..73518a4681d99 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -779,90 +779,6 @@ bb3: return %result : $Int64 } -// ----------------------------------------------------------------------------- -// Reduced test case from rdar://61246061 -// -// Test miscompilation of BidirectionalCollection._distance with -// combined load/store hoisting/sinking with multiple loads from -// aliasing addresses. - -// getRange -sil @getRange : $@convention(thin) () -> Range - -// CHECK-LABEL: sil shared @testLICMReducedCombinedLdStExtraProjection : $@convention(method) (Int64) -> Int64 { -// CHECK: bb0(%0 : $Int64): -// CHECK: store %0 to %{{.*}} : $*Int64 -// CHECK: load %{{.*}} : $*Int64 -// CHECK-NOT: {{(load|store)}} -// CHECK: bb7: -// CHECK-NEXT: store %{{.*}} to %{{.*}} : $*Int64 -// CHECK-NOT: {{(load|store)}} -// CHECK-LABEL: } // end sil function 'testLICMReducedCombinedLdStExtraProjection' -sil shared @testLICMReducedCombinedLdStExtraProjection : $@convention(method) (Int64) -> Int64 { -// %0 // users: %5, %1 -bb0(%0 : $Int64): - %1 = struct_extract %0 : $Int64, #Int64._value // users: %35, %20 - %2 = integer_literal $Builtin.Int64, 0 // user: %9 - %3 = alloc_stack $Index // users: %41, %13, %4 - %4 = struct_element_addr %3 : $*Index, #Index.value // users: %8, %5 - store %0 to %4 : $*Int64 // id: %5 - %6 = integer_literal $Builtin.Int64, 1 // user: %11 - %7 = integer_literal $Builtin.Int1, -1 // user: %11 - %8 = struct_element_addr %4 : $*Int64, #Int64._value // user: %34 - br bb1(%2 : $Builtin.Int64) // id: %9 - -// %10 // user: %11 -bb1(%10 : $Builtin.Int64): // Preds: bb8 bb0 - %11 = builtin "sadd_with_overflow_Int64"(%10 : $Builtin.Int64, %6 : $Builtin.Int64, %7 : $Builtin.Int1) : $(Builtin.Int64, Builtin.Int1) // user: %12 - %12 = tuple_extract %11 : $(Builtin.Int64, Builtin.Int1), 0 // users: %38, %37 - %13 = struct_element_addr %3 : $*Index, #Index.value // users: %32, %27, %24, %14 - %14 = struct_element_addr %13 : $*Int64, #Int64._value // user: %15 - %15 = load %14 : $*Builtin.Int64 // user: %18 - %16 = integer_literal $Builtin.Int64, 1 // user: %18 - %17 = integer_literal $Builtin.Int1, -1 // user: %18 - %18 = builtin "sadd_with_overflow_Int64"(%15 : $Builtin.Int64, %16 : $Builtin.Int64, %17 : $Builtin.Int1) : $(Builtin.Int64, Builtin.Int1) // user: %19 - %19 = tuple_extract %18 : $(Builtin.Int64, Builtin.Int1), 0 // users: %26, %23, %20 - %20 = builtin "cmp_eq_Int64"(%19 : $Builtin.Int64, %1 : $Builtin.Int64) : $Builtin.Int1 // user: %21 - cond_br %20, bb2, bb3 // id: %21 - -bb2: // Preds: bb1 - cond_br undef, bb4, bb5 // id: %22 - -bb3: // Preds: bb1 - %23 = struct $Int64 (%19 : $Builtin.Int64) // user: %24 - store %23 to %13 : $*Int64 // id: %24 - br bb6 // id: %25 - -bb4: // Preds: bb2 - %26 = struct $Int64 (%19 : $Builtin.Int64) // user: %27 - store %26 to %13 : $*Int64 // id: %27 - br bb6 // id: %28 - -bb5: // Preds: bb2 - // function_ref getRange - %29 = function_ref @getRange : $@convention(thin) () -> Range // user: %30 - %30 = apply %29() : $@convention(thin) () -> Range // user: %31 - %31 = struct_extract %30 : $Range, #Range.lowerBound // user: %32 - store %31 to %13 : $*Int64 // id: %32 - br bb6 // id: %33 - -bb6: // Preds: bb5 bb4 bb3 - %34 = load %8 : $*Builtin.Int64 // user: %35 - %35 = builtin "cmp_eq_Int64"(%34 : $Builtin.Int64, %1 : $Builtin.Int64) : $Builtin.Int1 // user: %36 - cond_br %35, bb7, bb8 // id: %36 - -bb7: // Preds: bb6 - br bb9(%12 : $Builtin.Int64) // id: %37 - -bb8: // Preds: bb6 - br bb1(%12 : $Builtin.Int64) // id: %38 - -// %39 // user: %40 -bb9(%39 : $Builtin.Int64): // Preds: bb7 - %40 = struct $Int64 (%39 : $Builtin.Int64) // user: %42 - dealloc_stack %3 : $*Index // id: %41 - return %40 : $Int64 // id: %42 -} // testConditionalTrapInInfiniteSyncLoop and // testConditionalTrapDominatingSyncLoopExit From dce2b7cfbaa36a1d730019329ea569b468f907ab Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Mon, 28 Jul 2025 11:38:33 +0100 Subject: [PATCH 12/44] Fix SSA updater not introducing phi nodes in hoistLoadsAndStores. --- .../LoopInvariantCodeMotion.swift | 74 +++++++++------- .../Sources/Optimizer/Utilities/Cloner.swift | 19 ++++- .../swift/SILOptimizer/OptimizerBridging.h | 1 + lib/SILOptimizer/Utils/OptimizerBridging.cpp | 4 + test/SILOptimizer/licm.sil | 84 +++++++++++++++++++ 5 files changed, 147 insertions(+), 35 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index a8f23305a8713..3b174ebc1ee85 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -21,7 +21,7 @@ let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion context: context ) } - context.removeTriviallyDeadInstructionsIgnoringDebugUses(in: function) +// context.removeTriviallyDeadInstructionsIgnoringDebugUses(in: function) } struct DiscoveredMovableInstructions { @@ -50,7 +50,7 @@ private func optimizeTopLevelLoop( repeat { guard - let movableInstructions = analyzeLoop( + var movableInstructions = analyzeLoop( loop: thisLoop, runsOnHighLevelSil: runsOnHighLevelSil, context: context @@ -59,11 +59,9 @@ private func optimizeTopLevelLoop( return // Encountered a loop without preheader. Return early. } - // TODO: Would it be a good idea to convert stacks to InstructionSets so that we have more efficient lookup? - thisLoopChanged = optimizeLoop( loop: thisLoop, - movableInstructions: movableInstructions, + movableInstructions: &movableInstructions, context: context ) } while thisLoopChanged @@ -371,7 +369,7 @@ private func analyzeLoop( private func optimizeLoop( loop: Loop, - movableInstructions: DiscoveredMovableInstructions, + movableInstructions: inout DiscoveredMovableInstructions, context: FunctionPassContext ) -> Bool { guard loop.preheader != nil else { @@ -380,7 +378,7 @@ private func optimizeLoop( if hoistAllLoadsAndStores( loop: loop, - movableInstructions: movableInstructions, + movableInstructions: &movableInstructions, context: context ) { return true // TODO: This is not very legible. We should break down this function or make a separate pass out of this. @@ -414,13 +412,13 @@ private func optimizeLoop( func hoistAllLoadsAndStores( loop: Loop, - movableInstructions: DiscoveredMovableInstructions, + movableInstructions: inout DiscoveredMovableInstructions, context: FunctionPassContext ) -> Bool { for accessPath in movableInstructions.loadAndStoreAddrs { hoistLoadsAndStores( loop: loop, - loadsAndStores: movableInstructions.loadsAndStores, + loadsAndStores: &movableInstructions.loadsAndStores, accessPath: accessPath, context: context ) @@ -441,7 +439,7 @@ func hoistAllLoadsAndStores( func hoistLoadsAndStores( loop: Loop, - loadsAndStores: [Instruction], + loadsAndStores: inout [Instruction], accessPath: AccessPath, context: FunctionPassContext ) { @@ -475,10 +473,9 @@ func hoistLoadsAndStores( var storeAddr: Value? for case let storeInst as StoreInst in loadsAndStores where isStore(storeInst, thatAccesses: accessPath) { - if let srcLoadInst = storeInst.source as? LoadInst { - guard isLoadWithAccessPath(srcLoadInst, thatOverlapsAccess: accessPath) else { - return // TODO: Do we really want't to just return without processing other instructions? - } + if let srcLoadInst = storeInst.source as? LoadInst, + isLoad(srcLoadInst, withinAccess: accessPath){ + return } if storeAddr == nil { @@ -517,6 +514,8 @@ func hoistLoadsAndStores( var currentBlock: BasicBlock? var currentVal: Value? + // This loop depends on loadsAndStores being + // in order the instructions appear in blocks! for inst in loadsAndStores { let block = inst.parentBlock @@ -527,12 +526,13 @@ func hoistLoadsAndStores( if let storeInst = inst as? StoreInst, isStore(storeInst, thatAccesses: accessPath) { currentVal = storeInst.source + loadsAndStores.removeAll(where: { $0 == storeInst }) context.erase(instruction: storeInst) continue } guard let loadInst = inst as? LoadInst, - isLoadWithAccessPath(loadInst, thatOverlapsAccess: accessPath) else { + isLoad(loadInst, withinAccess: accessPath) else { continue } @@ -550,6 +550,7 @@ func hoistLoadsAndStores( } loadInst.replace(with: projectedValue, context) + loadsAndStores.removeAll(where: { $0 == loadInst }) } for exitingOrLatchBlock in exitingAndLatchBlocks { @@ -1101,20 +1102,26 @@ private func isOnlyLoadedAndStored( stores: Stack, aliasAnalysis: AliasAnalysis ) -> Bool { - return !sideEffects + let a = !sideEffects .contains { sideEffect in sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) && !isStore(sideEffect, thatAccesses: accessPath) - && !isLoadWithAccessPath(sideEffect, thatOverlapsAccess: accessPath) - } && !loads + && !isLoad(sideEffect, withinAccess: accessPath) + } + + let b = !loads .contains { loadInst in loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) && !isLoadWithAccessPath(loadInst, thatOverlapsAccess: accessPath) - } && !stores + } + + let c = !stores .contains { storeInst in storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && !isStore(storeInst, thatAccesses: accessPath) } + + return a && b && c } private func isStore( @@ -1150,15 +1157,12 @@ private func isLoadWithAccessPath( thatOverlapsAccess accessPath: AccessPath ) -> Bool { guard let loadInst = inst as? LoadInst, - loadInst.loadOwnership != .take, // TODO: handle LoadOwnershipQualifier::Take - !loadInst.operand.value.accessPath.isEqualOrContains(accessPath), - (!accessPath.isEqualOrContains(loadInst.operand.value.accessPath) || - !loadInst.operand.value.accessPath.isEqualOrContains(accessPath)) + loadInst.loadOwnership != .take // TODO: handle LoadOwnershipQualifier::Take else { return false } - - return true + + return accessPath.isEqualOrContains(loadInst.operand.value.accessPath) || loadInst.operand.value.accessPath.isEqualOrContains(accessPath) } private func splitLoads( @@ -1168,11 +1172,15 @@ private func splitLoads( accessPath: AccessPath, context: FunctionPassContext ) -> Bool { + var tmpLoads = Stack(context) + defer { tmpLoads.deinitialize() } var splitCounter = 0 // TODO: Is the iterator created at the beggining of the loop immutable? - for loadInst in loads { + while let loadInst = loads.pop() { guard splitCounter <= 6 else { + tmpLoads.push(loadInst) + loads.append(contentsOf: tmpLoads) return false } @@ -1180,19 +1188,24 @@ private func splitLoads( loadInst.mayRead(fromAddress: storeAddr, context.aliasAnalysis), !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) else { + tmpLoads.push(loadInst) continue } - guard loadInst.accessPath.getProjection(to: accessPath)?.isMaterializable ?? false else { + let loadAccessPath = loadInst.operand.value.accessPath + guard !accessPath.isEqualOrContains(loadAccessPath) else { + tmpLoads.push(loadInst) continue } if let splitLoads: [LoadInst] = loadInst.trySplit(context) { splitCounter += splitLoads.count - loads.append(contentsOf: splitLoads) - movableInstructions.toDelete.insert(loadInst) + movableInstructions.loadsAndStores.replace([loadInst], with: splitLoads) + tmpLoads.append(contentsOf: splitLoads) } } + + loads.append(contentsOf: tmpLoads) return true } @@ -1278,21 +1291,18 @@ private func isCoveredByScope( extension AccessPath { fileprivate func isLoopInvariant(loop: Loop) -> Bool { - // %7 = pointer_to_address %1 : $Builtin.RawPointer to $*Index // user: %8 switch base { case .box(let inst as Instruction), .class(let inst as Instruction), .index(let inst as Instruction), .pointer(let inst as Instruction), .stack(let inst as Instruction), .storeBorrow(let inst as Instruction), .tail(let inst as Instruction): -// if loop.basicBlockSet.contains(inst.parentBlock) { if loop.basicBlocks.contains(inst.parentBlock) { return false } case .global, .argument: break case .yield(let beginApplyResult): -// if loop.basicBlockSet.contains(beginApplyResult.parentBlock) { if loop.basicBlocks.contains(beginApplyResult.parentBlock) { return false } diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift index 40e7ff3ce92e1..a0c60a3774869 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift @@ -90,7 +90,17 @@ struct Cloner { } mutating func cloneUseDefChain(addr: Value, checkBase: (Value) -> Bool) -> Value? { + // MARK: Hacky temp fix + if addr is AllocStackInst { + return nil + } + guard !checkBase(addr) else { + guard let inst = addr as? Instruction else { + return nil + } + + bridged.recordClonedInstruction(inst.bridged, inst.bridged) return addr } @@ -99,7 +109,6 @@ struct Cloner { // end_borrows or fix mark_dependence operands. case is BeginBorrowInst, is MarkDependenceInst: return nil case let singleValueInstruction as SingleValueInstruction: -// guard shouldClone(singleValueInstruction) else { return nil } // TODO: Double check whether correct guard let sourceOperand = singleValueInstruction.operands.first else { return nil } @@ -108,8 +117,8 @@ struct Cloner { } } - private func shouldClone(_ singleValueInstruction: SingleValueInstruction) -> Bool { - switch singleValueInstruction { + private func shouldClone(_ value: Value) -> Bool { + switch value { case is StructElementAddrInst, is TupleElementAddrInst, is IndexAddrInst, is TailAddrInst, is InitEnumDataAddrInst, is OpenExistentialAddrInst, is UncheckedTakeEnumDataAddrInst, is ProjectBoxInst, is ProjectBlockStorageInst, is MoveOnlyWrapperToCopyableAddrInst, is CopyableToMoveOnlyWrapperAddrInst, is MoveOnlyWrapperToCopyableBoxInst, is UncheckedAddrCastInst, is AddressToPointerInst, is PointerToAddressInst, is MarkUninitializedInst, is MarkUnresolvedReferenceBindingInst, is DropDeinitInst, is MarkUnresolvedReferenceBindingInst, is MarkDependenceInst, is CopyValueInst, is BeginBorrowInst, is StoreBorrowInst: return true default: return false } @@ -127,6 +136,10 @@ struct Cloner { return nil } +// for op in projectAddr.operands { +// _ = cloneUseDefChain(addr: op.value, checkBase: checkBase) +// } + let clone = clone(instruction: projectAddr) clone.setOperand(at: sourceOperand.index, to: projectedSource, context) return clone as! SingleValueInstruction diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index deb7b1d44638a..2c0706e80368e 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -233,6 +233,7 @@ struct BridgedCloner { void cloneFunctionBody(BridgedFunction originalFunction) const; SWIFT_IMPORT_UNSAFE BridgedValue getClonedValue(BridgedValue v); bool isValueCloned(BridgedValue v) const; + void recordClonedInstruction(BridgedInstruction origInst, BridgedInstruction clonedInst) const; BridgedInstruction clone(BridgedInstruction inst); }; diff --git a/lib/SILOptimizer/Utils/OptimizerBridging.cpp b/lib/SILOptimizer/Utils/OptimizerBridging.cpp index 8d37156304f27..29171a65440c8 100644 --- a/lib/SILOptimizer/Utils/OptimizerBridging.cpp +++ b/lib/SILOptimizer/Utils/OptimizerBridging.cpp @@ -680,6 +680,10 @@ bool BridgedCloner::isValueCloned(BridgedValue v) const { return cloner->isValueCloned(v.getSILValue()); } +void BridgedCloner::recordClonedInstruction(BridgedInstruction origInst, BridgedInstruction clonedInst) const { + cloner->recordClonedInstruction(origInst.unbridged(), clonedInst.unbridged()); +} + BridgedInstruction BridgedCloner::clone(BridgedInstruction inst) { return {cloner->cloneInst(inst.unbridged())->asSILNode()}; } diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index 73518a4681d99..e149624646a1d 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -779,6 +779,90 @@ bb3: return %result : $Int64 } +// ----------------------------------------------------------------------------- +// Reduced test case from rdar://61246061 +// +// Test miscompilation of BidirectionalCollection._distance with +// combined load/store hoisting/sinking with multiple loads from +// aliasing addresses. + +// getRange +sil @getRange : $@convention(thin) () -> Range + +// CHECK-LABEL: sil shared @testLICMReducedCombinedLdStExtraProjection : $@convention(method) (Int64) -> Int64 { +// CHECK: bb0(%0 : $Int64): +// CHECK: store %0 to %{{.*}} : $*Int64 +// CHECK: load %{{.*}} : $*Int64 +// CHECK-NOT: {{(load|store)}} +// CHECK: bb7: +// CHECK-NEXT: store %{{.*}} to %{{.*}} : $*Int64 +// CHECK-NOT: {{(load|store)}} +// CHECK-LABEL: } // end sil function 'testLICMReducedCombinedLdStExtraProjection' +sil shared @testLICMReducedCombinedLdStExtraProjection : $@convention(method) (Int64) -> Int64 { +// %0 // users: %5, %1 +bb0(%0 : $Int64): + %1 = struct_extract %0 : $Int64, #Int64._value // users: %35, %20 + %2 = integer_literal $Builtin.Int64, 0 // user: %9 + %3 = alloc_stack $Index // users: %41, %13, %4 + %4 = struct_element_addr %3 : $*Index, #Index.value // users: %8, %5 + store %0 to %4 : $*Int64 // id: %5 + %6 = integer_literal $Builtin.Int64, 1 // user: %11 + %7 = integer_literal $Builtin.Int1, -1 // user: %11 + %8 = struct_element_addr %4 : $*Int64, #Int64._value // user: %34 + br bb1(%2 : $Builtin.Int64) // id: %9 + +// %10 // user: %11 +bb1(%10 : $Builtin.Int64): // Preds: bb8 bb0 + %11 = builtin "sadd_with_overflow_Int64"(%10 : $Builtin.Int64, %6 : $Builtin.Int64, %7 : $Builtin.Int1) : $(Builtin.Int64, Builtin.Int1) // user: %12 + %12 = tuple_extract %11 : $(Builtin.Int64, Builtin.Int1), 0 // users: %38, %37 + %13 = struct_element_addr %3 : $*Index, #Index.value // users: %32, %27, %24, %14 + %14 = struct_element_addr %13 : $*Int64, #Int64._value // user: %15 + %15 = load %14 : $*Builtin.Int64 // user: %18 + %16 = integer_literal $Builtin.Int64, 1 // user: %18 + %17 = integer_literal $Builtin.Int1, -1 // user: %18 + %18 = builtin "sadd_with_overflow_Int64"(%15 : $Builtin.Int64, %16 : $Builtin.Int64, %17 : $Builtin.Int1) : $(Builtin.Int64, Builtin.Int1) // user: %19 + %19 = tuple_extract %18 : $(Builtin.Int64, Builtin.Int1), 0 // users: %26, %23, %20 + %20 = builtin "cmp_eq_Int64"(%19 : $Builtin.Int64, %1 : $Builtin.Int64) : $Builtin.Int1 // user: %21 + cond_br %20, bb2, bb3 // id: %21 + +bb2: // Preds: bb1 + cond_br undef, bb4, bb5 // id: %22 + +bb3: // Preds: bb1 + %23 = struct $Int64 (%19 : $Builtin.Int64) // user: %24 + store %23 to %13 : $*Int64 // id: %24 + br bb6 // id: %25 + +bb4: // Preds: bb2 + %26 = struct $Int64 (%19 : $Builtin.Int64) // user: %27 + store %26 to %13 : $*Int64 // id: %27 + br bb6 // id: %28 + +bb5: // Preds: bb2 + // function_ref getRange + %29 = function_ref @getRange : $@convention(thin) () -> Range // user: %30 + %30 = apply %29() : $@convention(thin) () -> Range // user: %31 + %31 = struct_extract %30 : $Range, #Range.lowerBound // user: %32 + store %31 to %13 : $*Int64 // id: %32 + br bb6 // id: %33 + +bb6: // Preds: bb5 bb4 bb3 + %34 = load %8 : $*Builtin.Int64 // user: %35 + %35 = builtin "cmp_eq_Int64"(%34 : $Builtin.Int64, %1 : $Builtin.Int64) : $Builtin.Int1 // user: %36 + cond_br %35, bb7, bb8 // id: %36 + +bb7: // Preds: bb6 + br bb9(%12 : $Builtin.Int64) // id: %37 + +bb8: // Preds: bb6 + br bb1(%12 : $Builtin.Int64) // id: %38 + +// %39 // user: %40 +bb9(%39 : $Builtin.Int64): // Preds: bb7 + %40 = struct $Int64 (%39 : $Builtin.Int64) // user: %42 + dealloc_stack %3 : $*Index // id: %41 + return %40 : $Int64 // id: %42 +} // testConditionalTrapInInfiniteSyncLoop and // testConditionalTrapDominatingSyncLoopExit From b2acc8a5939a6850cf881ac2f49f3290cdeddf2e Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Mon, 28 Jul 2025 12:47:05 +0100 Subject: [PATCH 13/44] Fix loop re-running for some transformation. Fix copying for values with e.g. function arguments as a base. --- .../LoopInvariantCodeMotion.swift | 27 ++++++++++++------- .../Sources/Optimizer/Utilities/Cloner.swift | 3 ++- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 3b174ebc1ee85..e373cb43fdbca 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -415,17 +415,18 @@ func hoistAllLoadsAndStores( movableInstructions: inout DiscoveredMovableInstructions, context: FunctionPassContext ) -> Bool { + var changed = false for accessPath in movableInstructions.loadAndStoreAddrs { - hoistLoadsAndStores( + changed = hoistLoadsAndStores( loop: loop, loadsAndStores: &movableInstructions.loadsAndStores, accessPath: accessPath, context: context - ) + ) || changed } guard !movableInstructions.toDelete.isEmpty else { - return false + return changed } for inst in movableInstructions.toDelete { @@ -442,7 +443,7 @@ func hoistLoadsAndStores( loadsAndStores: inout [Instruction], accessPath: AccessPath, context: FunctionPassContext -) { +) -> Bool { let exitingAndLatchBlocks = loop.exitingAndLatchBlocks guard storesCommonlyDominateLoopExits( @@ -451,7 +452,7 @@ func hoistLoadsAndStores( exitingBlocks: exitingAndLatchBlocks, context: context ) else { - return + return false } for exitingOrLatchBlock in exitingAndLatchBlocks { @@ -465,9 +466,10 @@ func hoistLoadsAndStores( } guard let preheader = loop.preheader else { - return + return false } + var changed = false let builder = Builder(before: preheader.terminator, context) var ssaUpdater: SSAUpdater? var storeAddr: Value? @@ -475,7 +477,7 @@ func hoistLoadsAndStores( for case let storeInst as StoreInst in loadsAndStores where isStore(storeInst, thatAccesses: accessPath) { if let srcLoadInst = storeInst.source as? LoadInst, isLoad(srcLoadInst, withinAccess: accessPath){ - return + return changed } if storeAddr == nil { @@ -487,14 +489,14 @@ func hoistLoadsAndStores( context ) } else if storeInst.destination.type != storeAddr!.type { - return + return changed } ssaUpdater?.addAvailableValue(storeInst.source, in: storeInst.parentBlock) } guard let storeAddr, var ssaUpdater else { - return + return changed } var cloner = Cloner(cloneBefore: preheader.terminator, context) @@ -503,7 +505,7 @@ func hoistLoadsAndStores( guard let initialAddr = (cloner.cloneUseDefChain(addr: storeAddr) { srcAddr in srcAddr.parentBlock.dominates(preheader, context.dominatorTree) }) else { - return + return changed } let ownership: LoadInst.LoadOwnership = preheader.terminator.parentFunction.hasOwnership ? .trivial : .unqualified @@ -528,6 +530,7 @@ func hoistLoadsAndStores( currentVal = storeInst.source loadsAndStores.removeAll(where: { $0 == storeInst }) context.erase(instruction: storeInst) + changed = true continue } @@ -551,6 +554,7 @@ func hoistLoadsAndStores( loadInst.replace(with: projectedValue, context) loadsAndStores.removeAll(where: { $0 == loadInst }) + changed = true } for exitingOrLatchBlock in exitingAndLatchBlocks { @@ -567,12 +571,15 @@ func hoistLoadsAndStores( destination: initialAddr, ownership: ownership ) + changed = true } } if initialLoad.isTriviallyDead { context.erase(instruction: initialLoad) } + + return changed } func projectLoadValue( diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift index a0c60a3774869..802fbd637a4e0 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift @@ -97,7 +97,8 @@ struct Cloner { guard !checkBase(addr) else { guard let inst = addr as? Instruction else { - return nil + // TODO: Might have to additionally register like the instruction below. + return addr } bridged.recordClonedInstruction(inst.bridged, inst.bridged) From 13d9b1eae729ed08d27269d7bca224838b8369cc Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Tue, 29 Jul 2025 11:32:31 +0100 Subject: [PATCH 14/44] Add recursive load spliting. --- .../LoopInvariantCodeMotion.swift | 2 +- .../Optimizer/Utilities/OptUtils.swift | 40 +++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index e373cb43fdbca..7a961a9799cae 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -1205,7 +1205,7 @@ private func splitLoads( continue } - if let splitLoads: [LoadInst] = loadInst.trySplit(context) { + if let splitLoads: [LoadInst] = loadInst.trySplit(context, recursive: true) { splitCounter += splitLoads.count movableInstructions.loadsAndStores.replace([loadInst], with: splitLoads) tmpLoads.append(contentsOf: splitLoads) diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift index 72c59326f731e..8c60e645fb7d8 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift @@ -601,14 +601,14 @@ extension StoreInst { extension LoadInst { @discardableResult - func trySplit(_ context: FunctionPassContext) -> Bool { - let arr: [LoadInst]? = trySplit(context) + func trySplit(_ context: FunctionPassContext, recursive: Bool = false) -> Bool { + let arr: [LoadInst]? = trySplit(context, recursive: recursive) return arr != nil } @_disfavoredOverload @discardableResult - func trySplit(_ context: FunctionPassContext) -> [LoadInst]? { + func trySplit(_ context: FunctionPassContext, recursive: Bool = false) -> [LoadInst]? { var elements = [LoadInst]() let builder = Builder(before: self, context) if type.isStruct { @@ -625,6 +625,23 @@ extension LoadInst { } let newStruct = builder.createStruct(type: self.type, elements: elements) self.replace(with: newStruct, context) + + if recursive { + elements = elements + .flatMap { splitLoad in + let fieldCount = splitLoad.type.getNominalFields( + in: parentFunction + )?.count ?? splitLoad.type.tupleElements.count + + if fieldCount > 1, + let recursiveSplitLoads: [LoadInst] = splitLoad.trySplit(context) { + return recursiveSplitLoads + } else { + return [splitLoad] + } + } + } + return elements } else if type.isTuple { let builder = Builder(before: self, context) @@ -635,6 +652,23 @@ extension LoadInst { } let newTuple = builder.createTuple(type: self.type, elements: elements) self.replace(with: newTuple, context) + + if recursive { + elements = elements + .flatMap { splitLoad in + let fieldCount = splitLoad.type.getNominalFields( + in: parentFunction + )?.count ?? splitLoad.type.tupleElements.count + + if fieldCount > 1, + let recursiveSplitLoads: [LoadInst] = splitLoad.trySplit(context) { + return recursiveSplitLoads + } else { + return [splitLoad] + } + } + } + return elements } return nil From fdccfca1c9eeb61e02963bcad7be4b0a0181689b Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Wed, 30 Jul 2025 15:47:11 +0100 Subject: [PATCH 15/44] Add trySplit along an access path. Other minor fixes. --- .../LoopInvariantCodeMotion.swift | 51 +++--- .../Optimizer/Utilities/OptUtils.swift | 104 +++++++----- .../SIL/Utilities/SmallProjectionPath.swift | 2 +- test/SILOptimizer/licm.sil | 156 +++++++++++++----- 4 files changed, 202 insertions(+), 111 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 7a961a9799cae..5a1c871730e41 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -17,7 +17,6 @@ let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion for loop in context.loopTree.loops { optimizeTopLevelLoop( topLevelLoop: loop, - runsOnHighLevelSil: true, // TODO: Make a parameter. context: context ) } @@ -37,8 +36,7 @@ struct DiscoveredMovableInstructions { private func optimizeTopLevelLoop( topLevelLoop: Loop, - runsOnHighLevelSil: Bool, - context: FunctionPassContext, + context: FunctionPassContext ) { var workList = getWorkList(topLevelLoop: topLevelLoop, context: context) defer { @@ -52,7 +50,6 @@ private func optimizeTopLevelLoop( guard var movableInstructions = analyzeLoop( loop: thisLoop, - runsOnHighLevelSil: runsOnHighLevelSil, context: context ) else { @@ -95,7 +92,6 @@ private func getWorkList(topLevelLoop: Loop, context: Context) -> Stack { private func analyzeLoop( loop: Loop, - runsOnHighLevelSil: Bool, context: FunctionPassContext ) -> DiscoveredMovableInstructions? { guard let preheader = loop.preheader else { @@ -206,7 +202,6 @@ private func analyzeLoop( case let applyInst as ApplyInst: if isSafeReadOnlyApply( applyInst: applyInst, - runsOnHighLevelSil: runsOnHighLevelSil, calleeAnalysis: context.calleeAnalysis ) { readOnlyApplies.push(applyInst) @@ -251,7 +246,6 @@ private func analyzeLoop( if canHoistUpDefault( inst: inst, loop: loop, - runsOnHighLevelSil: runsOnHighLevelSil, context: context ) { movableInstructions.hoistUp.append(inst) @@ -599,16 +593,26 @@ func projectLoadValue( } let builder = Builder(before: beforeInst, context) - let (kind, index, _) = projectionPath.pop() - switch kind { - case .structField: - return builder.createStructExtract(struct: rootVal, fieldIndex: index) - case .tupleField: - return builder.createTupleExtract(tuple: rootVal, elementIndex: index) - default: - return nil + var currPath = projectionPath + var currVal = rootVal + + while !currPath.isEmpty { + let (kind, index, remainderPath) = currPath.pop() + + switch kind { + case .structField: + currVal = builder.createStructExtract(struct: currVal, fieldIndex: index) + case .tupleField: + currVal = builder.createTupleExtract(tuple: currVal, elementIndex: index) + default: + return nil + } + + currPath = remainderPath } + + return currVal } func storesCommonlyDominateLoopExits( @@ -905,17 +909,14 @@ private func hasOwnershipOperandsOrResults(inst: Instruction) -> Bool { private func isSafeReadOnlyApply( applyInst: ApplyInst, - runsOnHighLevelSil: Bool, calleeAnalysis: CalleeAnalysis ) -> Bool { guard applyInst.functionConvention.results.allSatisfy({ $0.convention == .unowned }) else { return false } - if runsOnHighLevelSil, - let callee = applyInst.referencedFunction, - callee.hasSemanticsAttribute("array.props.isNativeTypeChecked") - { + if let callee = applyInst.referencedFunction, + callee.hasSemanticsAttribute("array.props.isNativeTypeChecked") { return false } @@ -925,7 +926,6 @@ private func isSafeReadOnlyApply( private func canHoistUpDefault( inst: Instruction, loop: Loop, - runsOnHighLevelSil: Bool, context: FunctionPassContext ) -> Bool { guard let preheader = loop.preheader else { @@ -938,15 +938,12 @@ private func canHoistUpDefault( switch inst.getArraySemanticsCallKind() { case .getCount, .getCapacity: - if runsOnHighLevelSil - && inst.canHoistArraySemanticsCall(to: preheader.terminator, context) + if inst.canHoistArraySemanticsCall(to: preheader.terminator, context) { return true } case .arrayPropsIsNativeTypeChecked: - if runsOnHighLevelSil { - return false - } + return false default: break } @@ -1205,7 +1202,7 @@ private func splitLoads( continue } - if let splitLoads: [LoadInst] = loadInst.trySplit(context, recursive: true) { + if let splitLoads: [LoadInst] = loadInst.trySplit(alongPath: accessPath.projectionPath, context) { splitCounter += splitLoads.count movableInstructions.loadsAndStores.replace([loadInst], with: splitLoads) tmpLoads.append(contentsOf: splitLoads) diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift index 8c60e645fb7d8..5ed53c02b76b3 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift @@ -601,22 +601,15 @@ extension StoreInst { extension LoadInst { @discardableResult - func trySplit(_ context: FunctionPassContext, recursive: Bool = false) -> Bool { - let arr: [LoadInst]? = trySplit(context, recursive: recursive) - return arr != nil - } - - @_disfavoredOverload - @discardableResult - func trySplit(_ context: FunctionPassContext, recursive: Bool = false) -> [LoadInst]? { + func trySplit(_ context: FunctionPassContext) -> Bool { var elements = [LoadInst]() let builder = Builder(before: self, context) if type.isStruct { if (type.nominal as! StructDecl).hasUnreferenceableStorage { - return nil + return false } guard let fields = type.getNominalFields(in: parentFunction) else { - return nil + return false } for idx in 0.. 1, - let recursiveSplitLoads: [LoadInst] = splitLoad.trySplit(context) { - return recursiveSplitLoads - } else { - return [splitLoad] - } - } - } - return elements + return true } else if type.isTuple { let builder = Builder(before: self, context) for idx in 0.. 1, - let recursiveSplitLoads: [LoadInst] = splitLoad.trySplit(context) { - return recursiveSplitLoads - } else { - return [splitLoad] - } - } - } - - return elements + return true } - return nil + return false } private func splitOwnership(for fieldValue: Value) -> LoadOwnership { @@ -682,6 +643,59 @@ extension LoadInst { return fieldValue.type.isTrivial(in: parentFunction) ? .trivial : self.loadOwnership } } + + func trySplit( + alongPath projectionPath: SmallProjectionPath, + _ context: FunctionPassContext + ) -> [LoadInst]? { + guard !projectionPath.isEmpty else { + return nil + } + + let (fieldKind, index, pathRemainder) = projectionPath.pop() + + var elements = [LoadInst]() + let builder = Builder(before: self, context) + + switch fieldKind { + case .structField where !(type.nominal as! StructDecl).hasUnreferenceableStorage && type.isStruct: + guard let fields = type.getNominalFields(in: parentFunction) else { + return nil + } + for idx in 0..) -> Int sil @user : $@convention(thin) (Int) -> () -// CHECK-LABEL: sil @dont_hoist_get_count_on_low_level_sil -// CHECK: {{^}}bb1: +// CHECK-LABEL: sil @hoist_get_count_on_low_level_sil +// CHECK: {{^}}bb0(%0 : $Array): // CHECK: apply +// CHECK: {{^}}bb1: // CHECK: apply // CHECK: {{^}}bb2: // CHECK: return -sil @dont_hoist_get_count_on_low_level_sil : $@convention(thin) (@guaranteed Array) -> () { +sil @hoist_get_count_on_low_level_sil : $@convention(thin) (@guaranteed Array) -> () { bb0(%0 : $Array): br bb1 @@ -943,26 +944,20 @@ bb5: return %99 : $() } -// Test load splitting with a loop-invariant stored value. The loop -// will be empty after combined load/store hoisting/sinking. +// Test load splitting with a loop-invariant stored value. `struct_element_addr` will be +// hoisted and will be possible to merge by subsequently running CSE pass. // -// TODO: sink a struct_extract (or other non-side-effect instructions) -// with no uses in the loop. -// -// CHECK-LABEL: sil shared @testLoadSplit : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { -// CHECK: [[PRELOAD:%.*]] = load %{{.*}} : $*Int64 -// CHECK: [[STOREDVAL:%.*]] = struct_extract %0 : $Int64, #Int64._value -// CHECK: br bb1([[PRELOAD]] : $Int64) -// CHECK: bb1([[PHI:%.*]] : $Int64): -// CHECK-NEXT: [[OUTERVAL:%.*]] = struct $Index ([[PHI]] : $Int64) -// CHECK-NEXT: cond_br undef, bb2, bb3 -// CHECK: bb2: -// CHECK-NEXT: br bb1(%0 : $Int64) -// CHECK: bb3: -// CHECK-NEXT: store %0 to %{{.*}} : $*Int64 -// CHECK-NEXT: tuple ([[OUTERVAL]] : $Index, [[PHI]] : $Int64, [[STOREDVAL]] : $Builtin.Int64) -// CHECK-LABEL: } // end sil function 'testLoadSplit' -sil shared @testLoadSplit : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +// CHECK-LABEL: sil shared @testLoadSplitBeforeCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +// CHECK: struct_element_addr +// CHECK: struct_element_addr +// CHECK: struct_element_addr +// CHECK: br bb1 +// CHECK: bb1: +// CHECK: load +// CHECK: load +// CHECK: store +// CHECK: load +sil shared @testLoadSplitBeforeCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { bb0(%0 : $Int64, %1 : $Builtin.RawPointer): %outerAddr1 = pointer_to_address %1 : $Builtin.RawPointer to $*Index %middleAddr1 = struct_element_addr %outerAddr1 : $*Index, #Index.value @@ -986,24 +981,63 @@ bb3: return %result : $(Index, Int64, Builtin.Int64) } +// Test load splitting with a loop-invariant stored value after CSE. The loop +// will be empty after combined load/store hoisting/sinking. +// +// CHECK-LABEL: sil shared @testLoadSplitAfterCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +// CHECK: [[PRELOAD:%.*]] = load %{{.*}} : $*Int64 +// CHECK: [[STOREDVAL:%.*]] = struct_extract %0 : $Int64, #Int64._value +// CHECK: br bb1([[PRELOAD]] : $Int64) +// CHECK: bb1([[PHI:%.*]] : $Int64): +// CHECK-NEXT: [[OUTERVAL:%.*]] = struct $Index ([[PHI]] : $Int64) +// CHECK-NEXT: cond_br undef, bb2, bb3 +// CHECK: bb2: +// CHECK-NEXT: br bb1(%0 : $Int64) +// CHECK: bb3: +// CHECK-NEXT: store %0 to %{{.*}} : $*Int64 +// CHECK-NEXT: tuple ([[OUTERVAL]] : $Index, [[PHI]] : $Int64, [[STOREDVAL]] : $Builtin.Int64) +// CHECK-LABEL: } // end sil function 'testLoadSplitAfterCSE' +sil shared @testLoadSplitAfterCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +bb0(%0 : $Int64, %1 : $Builtin.RawPointer): + %outerAddr = pointer_to_address %1 to $*Index + %middleAddr = struct_element_addr %outerAddr, #Index.value + %innerAddr = struct_element_addr %middleAddr, #Int64._value + br bb1 + +bb1: + %val1 = load %outerAddr + %val2 = load %middleAddr + store %0 to %middleAddr + %val3 = load %innerAddr + cond_br undef, bb2, bb3 + +bb2: + br bb1 + +bb3: + %result = tuple (%val1, %val2, %val3) + return %result +} // end sil function 'testLoadSplitAfterCSE' + // Test load splitting with a loop-varying stored value. -// CHECK-LABEL: sil shared @testLoadSplitPhi : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { -// CHECK: [[PRELOAD:%.*]] = load %{{.*}} : $*Int64 -// CHECK: br bb1(%4 : $Int64) -// CHECK: bb1([[PHI:%.*]] : $Int64): -// CHECK-NEXT: [[OUTERVAL:%.*]] = struct $Index ([[PHI]] : $Int64) -// CHECK-NEXT: [[EXTRACT:%.*]] = struct_extract [[PHI]] : $Int64, #Int64._value -// CHECK-NEXT: builtin "uadd_with_overflow_Int32"([[EXTRACT]] : $Builtin.Int64 -// CHECK-NEXT: tuple_extract -// CHECK-NEXT: [[ADD:%.*]] = struct $Int64 +// CHECK-LABEL: sil shared @testLoadSplitPhiBeforeCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +// CHECK: pointer_to_address +// CHECK: struct_element_addr +// CHECK: struct_element_addr +// CHECK: integer_literal +// CHECK: integer_literal +// CHECK: pointer_to_address +// CHECK: struct_element_addr +// CHECK: br bb1 +// CHECK: bb1: +// CHECK: load +// CHECK: load +// CHECK: load // CHECK-NEXT: cond_br undef, bb2, bb3 // CHECK: bb2: -// CHECK-NEXT: br bb1([[ADD]] : $Int64) -// CHECK: bb3: -// CHECK-NEXT: store [[ADD]] to %{{.*}} : $*Int64 -// CHECK-NEXT: tuple ([[OUTERVAL]] : $Index, [[ADD]] : $Int64, [[EXTRACT]] : $Builtin.Int64) -// CHECK-LABEL: } // end sil function 'testLoadSplitPhi' -sil shared @testLoadSplitPhi : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +// CHECK-NEXT: br bb1 +// CHECK-LABEL: } // end sil function 'testLoadSplitPhiBeforeCSE' +sil shared @testLoadSplitPhiBeforeCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { bb0(%0 : $Int64, %1 : $Builtin.RawPointer): %outerAddr1 = pointer_to_address %1 : $Builtin.RawPointer to $*Index %middleAddr1 = struct_element_addr %outerAddr1 : $*Index, #Index.value @@ -1032,6 +1066,51 @@ bb3: return %result : $(Index, Int64, Builtin.Int64) } +// Test load splitting with a loop-varying stored value. +// CHECK-LABEL: sil shared @testLoadSplitPhiAfterCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +// CHECK: [[PRELOAD:%.*]] = load %{{.*}} : $*Int64 +// CHECK: br bb1(%{{.*}} : $Int64) +// CHECK: bb1([[PHI:%.*]] : $Int64): +// CHECK-NEXT: [[OUTERVAL:%.*]] = struct $Index ([[PHI]] : $Int64) +// CHECK-NEXT: [[EXTRACT:%.*]] = struct_extract [[PHI]] : $Int64, #Int64._value +// CHECK-NEXT: builtin "uadd_with_overflow_Int32"([[EXTRACT]] : $Builtin.Int64 +// CHECK-NEXT: tuple_extract +// CHECK-NEXT: [[ADD:%.*]] = struct $Int64 +// CHECK-NEXT: cond_br undef, bb2, bb3 +// CHECK: bb2: +// CHECK-NEXT: br bb1([[ADD]] : $Int64) +// CHECK: bb3: +// CHECK-NEXT: store [[ADD]] to %{{.*}} : $*Int64 +// CHECK-NEXT: tuple ([[OUTERVAL]] : $Index, [[ADD]] : $Int64, [[EXTRACT]] : $Builtin.Int64) +// CHECK-LABEL: } // end sil function 'testLoadSplitPhiAfterCSE' +sil shared @testLoadSplitPhiAfterCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +// %1 // user: %2 +bb0(%0 : $Int64, %1 : $Builtin.RawPointer): + %2 = pointer_to_address %1 to $*Index // users: %8, %3 + %3 = struct_element_addr %2, #Index.value // users: %13, %14, %4 + %4 = struct_element_addr %3, #Int64._value // user: %9 + %5 = integer_literal $Builtin.Int64, 1 // user: %10 + %6 = integer_literal $Builtin.Int1, 0 // user: %10 + br bb1 // id: %7 + +bb1: // Preds: bb2 bb0 + %8 = load %2 // user: %17 + %9 = load %4 // users: %17, %10 + %10 = builtin "uadd_with_overflow_Int32"(%9, %5, %6) : $(Builtin.Int64, Builtin.Int1) // user: %11 + %11 = tuple_extract %10, 0 // user: %12 + %12 = struct $Int64 (%11) // user: %13 + store %12 to %3 // id: %13 + %14 = load %3 // user: %17 + cond_br undef, bb2, bb3 // id: %15 + +bb2: // Preds: bb1 + br bb1 // id: %16 + +bb3: // Preds: bb1 + %17 = tuple (%8, %14, %9) // user: %18 + return %17 // id: %18 +} // end sil function 'testLoadSplitPhiAfterCSE' + struct State { @_hasStorage var valueSet: (Int64, Int64, Int64) { get set } @_hasStorage var singleValue: Int64 { get set } @@ -1045,6 +1124,7 @@ struct State { // CHECK: bb0(%0 : $Builtin.RawPointer): // CHECK: [[HOISTADR:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 0 // ...Preload stored element #1 +// CHECK: tuple_element_addr // CHECK: [[PRELOADADR:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 1 // CHECK: [[PRELOAD:%.*]] = load [[PRELOADADR]] : $*Int64 // ...Split element 0 @@ -1294,7 +1374,7 @@ bb3: // CHECK: [[ELT_0:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 0 // CHECK: [[V0:%.*]] = load %6 : $*Int64 // CHECK: [[ARG0:%.*]] = tuple (%0 : $Int64, %0 : $Int64) -// CHECK: [[ARG0_0:%.*]] = tuple_extract %8 : $(Int64, Int64), 0 +// CHECK: [[ARG0_0:%.*]] = tuple_extract %10 : $(Int64, Int64), 0 // CHECK: [[ARG1:%.*]] = tuple (%1 : $Int64, %1 : $Int64) // CHECK: br bb1([[V1]] : $(Int64, Int64)) // CHECK: bb1([[PHI:%.*]] : $(Int64, Int64)): From 153df29734c826f65f961ff6c02e6ed32e913f29 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Wed, 30 Jul 2025 19:19:56 +0100 Subject: [PATCH 16/44] Refactor loop analysis. --- .../LoopInvariantCodeMotion.swift | 468 ++++++++---------- 1 file changed, 220 insertions(+), 248 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 5a1c871730e41..f0aabf003fcb2 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -12,26 +12,100 @@ import SIL -let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion") { - function, context in +let loopInvariantCodeMotionPass = FunctionPass( + name: "loop-invariant-code-motion" +) { function, context in for loop in context.loopTree.loops { optimizeTopLevelLoop( topLevelLoop: loop, context: context ) } -// context.removeTriviallyDeadInstructionsIgnoringDebugUses(in: function) } -struct DiscoveredMovableInstructions { - var toDelete: Set = [] - +struct MovableInstructions { + var loadAndStoreAddrs: [AccessPath] = [] + var loadsAndStores: [Instruction] = [] var hoistUp: [Instruction] = [] var sinkDown: [Instruction] = [] var specialHoist: [Instruction] = [] +} - var loadAndStoreAddrs: [AccessPath] = [] +struct AnalyzedInstructions { + private(set) var globalInitCalls: Stack + private(set) var loopSideEffects: Stack + + private(set) var readOnlyApplies: Stack + private(set) var loads: Stack + private(set) var stores: Stack + private(set) var fixLifetimes: Stack + private(set) var beginAccesses: Stack + private(set) var fullApplies: Stack + + private(set) var readOnlyAppliesCount = 0 + private(set) var loopSideEffectCount = 0 + private(set) var loadsCount = 0 + + init (_ context: FunctionPassContext) { + self.globalInitCalls = Stack(context) + self.loopSideEffects = Stack(context) + + self.readOnlyApplies = Stack(context) + self.loads = Stack(context) + self.stores = Stack(context) + self.fixLifetimes = Stack(context) + self.beginAccesses = Stack(context) + self.fullApplies = Stack(context) + } + + mutating func appendGlobalInitCall(_ inst: Instruction) { globalInitCalls.append(inst) } + + mutating func appendSideEffect(_ inst: Instruction) { + loopSideEffects.append(inst) + loopSideEffectCount += 1 + } + + mutating func append(_ applyInst: ApplyInst) { + readOnlyApplies.append(applyInst) + readOnlyAppliesCount += 1 + } + + mutating func append(_ loadInst: LoadInst) { + loads.append(loadInst) + loadsCount += 1 + } + + mutating func popLoad() -> LoadInst? { + guard !loads.isEmpty else { + return nil + } + + loadsCount -= 1 + return loads.pop() + } + + mutating func append(newLoads: Stack) { + for load in newLoads { + append(load) + } + } + + mutating func append(_ storeInst: StoreInst) { stores.append(storeInst) } + mutating func append(_ fixLifetime: FixLifetimeInst) { fixLifetimes.append(fixLifetime) } + mutating func append(_ beginAccess: BeginAccessInst) { beginAccesses.append(beginAccess) } + mutating func append(_ fullApply: FullApplySite) { fullApplies.append(fullApply) } + + mutating func deinitialize() { + readOnlyApplies.deinitialize() + globalInitCalls.deinitialize() + loopSideEffects.deinitialize() + loads.deinitialize() + stores.deinitialize() + fixLifetimes.deinitialize() + beginAccesses.deinitialize() + fullApplies.deinitialize() + } } private func optimizeTopLevelLoop( @@ -93,72 +167,42 @@ private func getWorkList(topLevelLoop: Loop, context: Context) -> Stack { private func analyzeLoop( loop: Loop, context: FunctionPassContext -) -> DiscoveredMovableInstructions? { +) -> MovableInstructions? { guard let preheader = loop.preheader else { return nil } - var movableInstructions = DiscoveredMovableInstructions() - - var readOnlyApplies = Stack(context) - var globalInitCalls = Stack(context) - - var loopSideEffects = Stack(context) - var loads = Stack(context) - var stores = Stack(context) - var fixLifetimes = Stack(context) - var beginAccesses = Stack(context) - var fullApplies = Stack(context) - defer { - readOnlyApplies.deinitialize() - globalInitCalls.deinitialize() - loopSideEffects.deinitialize() - loads.deinitialize() - stores.deinitialize() - fixLifetimes.deinitialize() - beginAccesses.deinitialize() - fullApplies.deinitialize() - } - - // The stack data structure doesn't keep track of element count. We need to maintain those counts manually. - var readOnlyAppliesCount = 0 - var loopSideEffectCount = 0 - var loadsCount = 0 + var movableInstructions = MovableInstructions() + var analyzedInstructions = AnalyzedInstructions(context) + defer { analyzedInstructions.deinitialize() } var hasOtherMemReadingInsts = false for bb in loop.basicBlocks { var blockSideEffects = Stack(context) - defer { - blockSideEffects.deinitialize() - } + defer { blockSideEffects.deinitialize() } for inst in bb.instructions { - if hasOwnershipOperandsOrResults(inst: inst) { - if checkSideEffects( - inst: inst, - loopSideEffects: &loopSideEffects, + if hasOwnershipOperandsOrResults(inst) { + checkSideEffects( + of: inst, + analyzedInstructions: &analyzedInstructions, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) { - loopSideEffectCount += 1 + ) + + if let fullApply = inst as? FullApplySite { + analyzedInstructions.append(fullApply) } - - guard let fullApply = inst as? FullApplySite else { continue } - - fullApplies.push(fullApply) continue } switch inst { - case let fixLifetimeInst as FixLifetimeInst: - if fixLifetimeInst.parentBlock.dominates(preheader, context.dominatorTree) { - fixLifetimes.push(fixLifetimeInst) - } + case let fixLifetimeInst as FixLifetimeInst where fixLifetimeInst.parentBlock.dominates(preheader, context.dominatorTree): + analyzedInstructions.append(fixLifetimeInst) case let loadInst as LoadInst: - loads.push(loadInst) - loadsCount += 1 + analyzedInstructions.append(loadInst) movableInstructions.loadsAndStores.append(loadInst) case let storeInst as StoreInst: switch storeInst.storeOwnership { @@ -167,81 +211,74 @@ private func analyzeLoop( case .unqualified, .trivial: break } - stores.push(storeInst) + analyzedInstructions.append(storeInst) movableInstructions.loadsAndStores.append(storeInst) - if checkSideEffects( - inst: storeInst, - loopSideEffects: &loopSideEffects, + checkSideEffects( + of: storeInst, + analyzedInstructions: &analyzedInstructions, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) { - loopSideEffectCount += 1 - } + ) case let beginAccessInst as BeginAccessInst: - beginAccesses.push(beginAccessInst) - if checkSideEffects( - inst: beginAccessInst, - loopSideEffects: &loopSideEffects, + analyzedInstructions.append(beginAccessInst) + checkSideEffects( + of: beginAccessInst, + analyzedInstructions: &analyzedInstructions, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) { - loopSideEffectCount += 1 - } + ) case let refElementAddrInst as RefElementAddrInst: movableInstructions.specialHoist.append(refElementAddrInst) case let condFailInst as CondFailInst: movableInstructions.hoistUp.append(condFailInst) - if checkSideEffects( - inst: condFailInst, - loopSideEffects: &loopSideEffects, + checkSideEffects( + of: condFailInst, + analyzedInstructions: &analyzedInstructions, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) { - loopSideEffectCount += 1 - } + ) case let applyInst as ApplyInst: if isSafeReadOnlyApply( applyInst: applyInst, calleeAnalysis: context.calleeAnalysis ) { - readOnlyApplies.push(applyInst) - readOnlyAppliesCount += 1 + analyzedInstructions.append(applyInst) } else if let callee = applyInst.referencedFunction, callee.isGlobalInitFunction, !mayConflictWithGlobalInit( - globalInitCall: applyInst, sideEffects: blockSideEffects, - aliasAnalysis: context.aliasAnalysis) - { - globalInitCalls.push(applyInst) + globalInitCall: applyInst, + sideEffects: blockSideEffects, + aliasAnalysis: context.aliasAnalysis + ) { + analyzedInstructions.appendGlobalInitCall(applyInst) } fallthrough default: switch inst { case let fullApply as FullApplySite: - fullApplies.push(fullApply) + analyzedInstructions.append(fullApply) case let builtinInst as BuiltinInst: switch builtinInst.id { case .Once, .OnceWithContext: if !mayConflictWithGlobalInit( - globalInitCall: builtinInst, sideEffects: blockSideEffects, - aliasAnalysis: context.aliasAnalysis) - { - globalInitCalls.push(builtinInst) + globalInitCall: builtinInst, + sideEffects: blockSideEffects, + aliasAnalysis: context.aliasAnalysis + ) { + analyzedInstructions.appendGlobalInitCall(builtinInst) } default: break } default: break } - if checkSideEffects( - inst: inst, - loopSideEffects: &loopSideEffects, + checkSideEffects( + of: inst, + analyzedInstructions: &analyzedInstructions, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) { - loopSideEffectCount += 1 - } + ) if canHoistUpDefault( inst: inst, @@ -255,11 +292,11 @@ private func analyzeLoop( } // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. - if readOnlyAppliesCount * loopSideEffectCount < 8000 { - for readOnlyApply in readOnlyApplies { + if analyzedInstructions.readOnlyAppliesCount * analyzedInstructions.loopSideEffectCount < 8000 { + for readOnlyApply in analyzedInstructions.readOnlyApplies { if !mayWriteTo( readOnlyApply, - sideEffects: loopSideEffects, + sideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis, calleeAnalysis: context.calleeAnalysis ) { @@ -269,11 +306,11 @@ private func analyzeLoop( } // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. - if loadsCount * loopSideEffectCount < 8000 { - for load in loads { + if analyzedInstructions.loadsCount * analyzedInstructions.loopSideEffectCount < 8000 { + for load in analyzedInstructions.loads { if !mayWriteTo( load, - sideEffects: loopSideEffects, + sideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis ) { movableInstructions.hoistUp.append(load) @@ -281,14 +318,12 @@ private func analyzeLoop( } } - if !globalInitCalls.isEmpty { - // Pre check for post dom tree root node MIGHT not be necessary. - - for globalInitCall in globalInitCalls { + if !analyzedInstructions.globalInitCalls.isEmpty { + for globalInitCall in analyzedInstructions.globalInitCalls { if !mayConflictWithGlobalInit( globalInitCall: globalInitCall, preheader: preheader, - sideEffects: loopSideEffects, + sideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis, postDomTree: context.postDominatorTree ) { @@ -298,62 +333,56 @@ private func analyzeLoop( } if !hasOtherMemReadingInsts { - for storeInst in stores { + for storeInst in analyzedInstructions.stores { let accessPath = storeInst.destination.accessPath if accessPath.isLoopInvariant(loop: loop), isOnlyLoadedAndStored( accessPath: accessPath, storeAddr: storeInst.destination, - sideEffects: loopSideEffects, - loads: loads, - stores: stores, + analyzedInstructions: analyzedInstructions, aliasAnalysis: context.aliasAnalysis ), !movableInstructions.loadAndStoreAddrs.contains(accessPath), splitLoads( - loads: &loads, + analyzedInstructions: &analyzedInstructions, storeAddr: storeInst.destination, movableInstructions: &movableInstructions, accessPath: accessPath, context: context - ) - { + ) { movableInstructions.loadAndStoreAddrs.append(accessPath) } } } - if !fixLifetimes.isEmpty { - let sideEffectsMayRelease = loopSideEffects.contains(where: { $0.mayRelease }) + if !analyzedInstructions.fixLifetimes.isEmpty { + let sideEffectsMayRelease = analyzedInstructions.loopSideEffects.contains(where: { $0.mayRelease }) - for fixLifetime in fixLifetimes { + for fixLifetime in analyzedInstructions.fixLifetimes { guard fixLifetime.operand.value.type.isAddress else { continue } - if sideEffectsMayRelease - || !mayWriteTo( - fixLifetime, sideEffects: loopSideEffects, aliasAnalysis: context.aliasAnalysis) - { + if sideEffectsMayRelease || !mayWriteTo( + fixLifetime, + sideEffects: analyzedInstructions.loopSideEffects, + aliasAnalysis: context.aliasAnalysis + ) { movableInstructions.sinkDown.append(fixLifetime) } } } - for beginAccessInst in beginAccesses { + for beginAccessInst in analyzedInstructions.beginAccesses { if handledEndAccess( beginAccessInst: beginAccessInst, loop: loop, context: context - ) - && analyzeBeginAccess( - beginAccessInst: beginAccessInst, - beginAccesses: beginAccesses, - fullApplies: fullApplies, - sideEffects: loopSideEffects, - aliasAnalysis: context.aliasAnalysis, - domTree: context.dominatorTree - ) - { + ) && analyzeBeginAccess( + beginAccessInst: beginAccessInst, + analyzedInstructions: analyzedInstructions, + aliasAnalysis: context.aliasAnalysis, + domTree: context.dominatorTree + ) { movableInstructions.specialHoist.append(beginAccessInst) } } @@ -363,7 +392,7 @@ private func analyzeLoop( private func optimizeLoop( loop: Loop, - movableInstructions: inout DiscoveredMovableInstructions, + movableInstructions: inout MovableInstructions, context: FunctionPassContext ) -> Bool { guard loop.preheader != nil else { @@ -375,7 +404,7 @@ private func optimizeLoop( movableInstructions: &movableInstructions, context: context ) { - return true // TODO: This is not very legible. We should break down this function or make a separate pass out of this. + return true } var changed = false @@ -406,7 +435,7 @@ private func optimizeLoop( func hoistAllLoadsAndStores( loop: Loop, - movableInstructions: inout DiscoveredMovableInstructions, + movableInstructions: inout MovableInstructions, context: FunctionPassContext ) -> Bool { var changed = false @@ -419,17 +448,7 @@ func hoistAllLoadsAndStores( ) || changed } - guard !movableInstructions.toDelete.isEmpty else { - return changed - } - - for inst in movableInstructions.toDelete { - if inst.isTriviallyDead { - context.erase(instruction: inst) - } - } - - return true + return changed } func hoistLoadsAndStores( @@ -470,7 +489,7 @@ func hoistLoadsAndStores( for case let storeInst as StoreInst in loadsAndStores where isStore(storeInst, thatAccesses: accessPath) { if let srcLoadInst = storeInst.source as? LoadInst, - isLoad(srcLoadInst, withinAccess: accessPath){ + isLoad(srcLoadInst, withinAccess: accessPath) { return changed } @@ -673,9 +692,7 @@ func hoistInstructions( } var dominatingBlocks = getDominatingBlocks(loop: loop, context: context) - defer { - dominatingBlocks.deinitialize() - } + defer { dominatingBlocks.deinitialize() } var changed = false for bb in dominatingBlocks { @@ -699,13 +716,7 @@ private func hoistInstruction( loop: Loop, context: FunctionPassContext ) -> Bool { - // Check whether inst is loop invariant. - guard - (inst.operands.allSatisfy { operand in - !loop.basicBlocks.contains(operand.value.parentBlock) -// !loop.basicBlockSet.contains(operand.value.parentBlock) - }) - else { + guard inst.operands.allSatisfy({ !loop.basicBlocks.contains($0.value.parentBlock) }) else { return false } @@ -725,22 +736,15 @@ private func sinkInstructions( context: FunctionPassContext ) -> Bool { var dominatingBlocks = getDominatingBlocks(loop: loop, context: context) - defer { - dominatingBlocks.deinitialize() - } + defer { dominatingBlocks.deinitialize() } var changed = false - for inst in sinkDown { - guard dominatingBlocks.contains(inst.parentBlock) else { - continue - } - - changed = - sinkInstruction( - loop: loop, - inst: inst, - context: context - ) || changed + for inst in sinkDown where dominatingBlocks.contains(inst.parentBlock) { + changed = sinkInstruction( + loop: loop, + inst: inst, + context: context + ) || changed } return changed @@ -755,28 +759,22 @@ private func sinkInstruction( let exitBlocks = loop.exitBlocks let exitingBlocks = loop.exitingBlocks var newExitBlocks = Stack(context) - defer { - newExitBlocks.deinitialize() - } - + defer { newExitBlocks.deinitialize() } var changed = false for exitingBlock in exitingBlocks { for (succesorIndex, succesor) in exitingBlock.successors.enumerated().reversed() where !newExitBlocks.contains(succesor) && exitBlocks.contains(succesor) { - let outsideBlock = - context.loopTree.splitCriticalEdge( - basicBlock: exitingBlock, - edgeIndex: succesorIndex, - domTree: context.dominatorTree - ) ?? succesor + let outsideBlock = context.loopTree.splitCriticalEdge( + basicBlock: exitingBlock, + edgeIndex: succesorIndex, + domTree: context.dominatorTree + ) ?? succesor newExitBlocks.push(outsideBlock) - if (outsideBlock.instructions.contains { otherInst in - inst.isIdenticalTo(otherInst) - }) { + if outsideBlock.instructions.contains(where: { inst.isIdenticalTo($0) }) { isSingleExit = false } else if isSingleExit, let firstInstruction = outsideBlock.instructions.first { inst.move(before: firstInstruction, context) @@ -797,7 +795,6 @@ private func sinkInstruction( return changed } -// TODO: Give it a better name. private func hoistSpecialInstruction( loop: Loop, specialInsts: [Instruction], @@ -827,9 +824,7 @@ private func hoistSpecialInstruction( // TODO: This should probably be moved to the hoistUp collection. We should keep special hoist to only BeginAccessInst. if let beginAccessInst = specialInst as? BeginAccessInst { var endAccesses = getEndAccesses(beginAccessInst: beginAccessInst, context: context) - defer { - endAccesses.deinitialize() - } + defer { endAccesses.deinitialize() } for endAccess in endAccesses { _ = sinkInstruction(loop: loop, inst: endAccess, context: context) @@ -884,23 +879,20 @@ private func getDominatingBlocksHelper( /// Returns `true` if `inst` may have side effects. private func checkSideEffects( - inst: Instruction, - loopSideEffects: inout Stack, + of inst: Instruction, + analyzedInstructions: inout AnalyzedInstructions, blockSideEffects: inout Stack, hasOtherMemReadingInsts: inout Bool -) -> Bool { +) { if inst.mayHaveSideEffects { - loopSideEffects.push(inst) - blockSideEffects.push(inst) - return true + analyzedInstructions.appendSideEffect(inst) + blockSideEffects.append(inst) } else if inst.mayReadFromMemory { hasOtherMemReadingInsts = true } - - return false } -private func hasOwnershipOperandsOrResults(inst: Instruction) -> Bool { +private func hasOwnershipOperandsOrResults(_ inst: Instruction) -> Bool { guard inst.parentFunction.hasOwnership else { return false } return inst.results.contains(where: { $0.ownership != .none }) @@ -1010,25 +1002,25 @@ private func mayWriteTo( } } -private func handledEndAccess(beginAccessInst: BeginAccessInst, loop: Loop, context: Context) - -> Bool -{ +private func handledEndAccess( + beginAccessInst: BeginAccessInst, + loop: Loop, + context: Context +) -> Bool { var endAccesses = getEndAccesses(beginAccessInst: beginAccessInst, context: context) - defer { - endAccesses.deinitialize() - } + defer { endAccesses.deinitialize() } return !endAccesses.isEmpty && !endAccesses .contains { user in !loop.basicBlocks.contains(user.parentBlock) -// !loop.basicBlockSet.contains(user.parentBlock) } } -private func getEndAccesses(beginAccessInst: BeginAccessInst, context: Context) -> Stack< - EndAccessInst -> { +private func getEndAccesses( + beginAccessInst: BeginAccessInst, + context: Context +) -> Stack { var endAccesses = Stack(context) endAccesses.append( @@ -1061,8 +1053,7 @@ private func mayConflictWithGlobalInit( sideEffects: Stack, aliasAnalysis: AliasAnalysis ) -> Bool { - return - sideEffects + return sideEffects .contains { sideEffect in mayConflictWithGlobalInit( globalInitCall: globalInitCall, @@ -1083,49 +1074,39 @@ private func mayConflictWithGlobalInit( return true } - return - sideEffects + return sideEffects .contains { sideEffect in globalInitCall.parentBlock.strictlyPostDominates( sideEffect.parentBlock, postDomTree + ) && mayConflictWithGlobalInit( + globalInitCall: globalInitCall, + sideEffect: sideEffect, + aliasAnalysis: aliasAnalysis ) - && mayConflictWithGlobalInit( - globalInitCall: globalInitCall, - sideEffect: sideEffect, - aliasAnalysis: aliasAnalysis - ) } } private func isOnlyLoadedAndStored( accessPath: AccessPath, storeAddr: Value, - sideEffects: Stack, - loads: Stack, - stores: Stack, + analyzedInstructions: AnalyzedInstructions, aliasAnalysis: AliasAnalysis ) -> Bool { - let a = !sideEffects + return !analyzedInstructions.loopSideEffects .contains { sideEffect in sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) && !isStore(sideEffect, thatAccesses: accessPath) && !isLoad(sideEffect, withinAccess: accessPath) - } - - let b = !loads + } && !analyzedInstructions.loads .contains { loadInst in loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) && !isLoadWithAccessPath(loadInst, thatOverlapsAccess: accessPath) - } - - let c = !stores + } && !analyzedInstructions.stores .contains { storeInst in storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && !isStore(storeInst, thatAccesses: accessPath) } - - return a && b && c } private func isStore( @@ -1152,7 +1133,6 @@ private func isLoad( return false } - // TODO: Check if this is sufficient return accessPath.getProjection(to: loadInst.address.accessPath)?.isMaterializable ?? false } @@ -1170,9 +1150,9 @@ private func isLoadWithAccessPath( } private func splitLoads( - loads: inout Stack, + analyzedInstructions: inout AnalyzedInstructions, storeAddr: Value, - movableInstructions: inout DiscoveredMovableInstructions, + movableInstructions: inout MovableInstructions, accessPath: AccessPath, context: FunctionPassContext ) -> Bool { @@ -1180,15 +1160,14 @@ private func splitLoads( defer { tmpLoads.deinitialize() } var splitCounter = 0 - // TODO: Is the iterator created at the beggining of the loop immutable? - while let loadInst = loads.pop() { + while let loadInst = analyzedInstructions.popLoad() { guard splitCounter <= 6 else { tmpLoads.push(loadInst) - loads.append(contentsOf: tmpLoads) + analyzedInstructions.append(newLoads: tmpLoads) return false } - guard !movableInstructions.toDelete.contains(loadInst), + guard !loadInst.isDeleted, loadInst.mayRead(fromAddress: storeAddr, context.aliasAnalysis), !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) else { @@ -1209,21 +1188,18 @@ private func splitLoads( } } - loads.append(contentsOf: tmpLoads) + analyzedInstructions.append(newLoads: tmpLoads) return true } private func analyzeBeginAccess( beginAccessInst: BeginAccessInst, - beginAccesses: Stack, - fullApplies: Stack, - sideEffects: Stack, + analyzedInstructions: AnalyzedInstructions, aliasAnalysis: AliasAnalysis, domTree: DominatorTree ) -> Bool { - let areBeginAccessesSafe = - beginAccesses + let areBeginAccessesSafe = analyzedInstructions.beginAccesses .allSatisfy { otherBeginAccessInst in guard beginAccessInst != otherBeginAccessInst else { return true } @@ -1232,18 +1208,15 @@ private func analyzeBeginAccess( guard areBeginAccessesSafe else { return false } - for fullApplyInst in fullApplies { - guard - beginAccessInst.mayWriteToMemory + for fullApplyInst in analyzedInstructions.fullApplies { + guard beginAccessInst.mayWriteToMemory ? fullApplyInst.mayReadOrWrite( address: beginAccessInst.address, aliasAnalysis - ) - : fullApplyInst.mayWrite( + ) : fullApplyInst.mayWrite( toAddress: beginAccessInst.address, aliasAnalysis - ) - else { + ) else { continue } @@ -1258,7 +1231,7 @@ private func analyzeBeginAccess( switch beginAccessInst.accessPath.base { case .class, .global: - for sideEffect in sideEffects { + for sideEffect in analyzedInstructions.loopSideEffects { guard sideEffect.mayRelease else { continue } @@ -1286,11 +1259,10 @@ private func isCoveredByScope( return beginAccessInst.parentBlock.strictlyDominates( otherInst.parentBlock, domTree - ) - && beginAccessInst.endAccessInstructions - .allSatisfy { endAccessInst in - otherInst.parentBlock.dominates(endAccessInst.parentBlock, domTree) - } + ) && beginAccessInst.endAccessInstructions + .allSatisfy { endAccessInst in + otherInst.parentBlock.dominates(endAccessInst.parentBlock, domTree) + } } extension AccessPath { From cf5b9b885876b6bcf66d3e283860471f5b4fdc0c Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Thu, 31 Jul 2025 12:41:22 +0100 Subject: [PATCH 17/44] Restructure licm algorithm into extensions. --- .../Sources/Optimizer/Analysis/LoopTree.swift | 32 +- .../LoopInvariantCodeMotion.swift | 1536 ++++++++--------- 2 files changed, 731 insertions(+), 837 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index d3a1b491b56a6..7eb59f858802e 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -72,7 +72,6 @@ struct Loop { return header.predecessors.lazy .filter { predecessor in basicBlocks.contains(predecessor) && !isLoopExiting(bb: predecessor) -// basicBlockSet.contains(predecessor) && !isLoopExiting(bb: predecessor) } + exitingBlocks } @@ -81,7 +80,6 @@ struct Loop { .flatMap(\.successors) .filter { succesor in !basicBlocks.contains(succesor) -// !basicBlockSet.contains(succesor) } } @@ -104,7 +102,34 @@ struct Loop { return bb.successors .contains { succesor in !basicBlocks.contains(succesor) -// !basicBlockSet.contains(succesor) + } + } + + func getBlocksThatDominateAllExitingAndLatchBlocks( + domTree: DominatorTree + ) -> some Sequence { + return getBlocksThatDominateAllExitingAndLatchBlocksHelper( + bb: header, + domTree: domTree + ) + } + + private func getBlocksThatDominateAllExitingAndLatchBlocksHelper( + bb: BasicBlock, + domTree: DominatorTree + ) -> some Sequence { + guard exitingAndLatchBlocks.allSatisfy({ exitBlock in + return bb.dominates(exitBlock, domTree) + }) else { + return [] + } + + return [bb] + domTree.getChildren(of: bb).lazy + .flatMap { child in + getBlocksThatDominateAllExitingAndLatchBlocksHelper( + bb: child, + domTree: domTree + ) } } @@ -113,7 +138,6 @@ struct Loop { self.bridged = bridged self.innerLoops = LoopArray(bridged, context: context) self.basicBlocks = BasicBlockArray(bridged) -// self.basicBlockSet = BasicBlockSet(insertContentsOf: basicBlocks, context) } } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index f0aabf003fcb2..e9425efbd43fc 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -23,6 +23,60 @@ let loopInvariantCodeMotionPass = FunctionPass( } } +private func optimizeTopLevelLoop( + topLevelLoop: Loop, + context: FunctionPassContext +) { + var workList = getWorkList(topLevelLoop: topLevelLoop, context: context) + defer { workList.deinitialize() } + + while let thisLoop = workList.pop() { + var thisLoopChanged = false + + repeat { + guard + var movableInstructions = analyzeLoop( + loop: thisLoop, + context: context + ) + else { + return // Encountered a loop without preheader. Return early. + } + + thisLoopChanged = optimizeLoop( + loop: thisLoop, + movableInstructions: &movableInstructions, + context: context + ) + } while thisLoopChanged + } +} + +private func getWorkList(topLevelLoop: Loop, context: Context) -> Stack { + var tmp1 = Stack(context) + var tmp2 = Stack(context) + var workList = Stack(context) + defer { + tmp1.deinitialize() + tmp2.deinitialize() + } + + tmp1.push(topLevelLoop) + + while !tmp1.isEmpty || !tmp2.isEmpty { + while let loop = tmp2.pop() { + workList.push(loop) + } + + while let loop = tmp1.pop() { + tmp2.push(loop) + tmp1.append(contentsOf: loop.innerLoops) + } + } + + return workList +} + struct MovableInstructions { var loadAndStoreAddrs: [AccessPath] = [] @@ -108,62 +162,6 @@ struct AnalyzedInstructions { } } -private func optimizeTopLevelLoop( - topLevelLoop: Loop, - context: FunctionPassContext -) { - var workList = getWorkList(topLevelLoop: topLevelLoop, context: context) - defer { - workList.deinitialize() - } - - while let thisLoop = workList.pop() { - var thisLoopChanged = false - - repeat { - guard - var movableInstructions = analyzeLoop( - loop: thisLoop, - context: context - ) - else { - return // Encountered a loop without preheader. Return early. - } - - thisLoopChanged = optimizeLoop( - loop: thisLoop, - movableInstructions: &movableInstructions, - context: context - ) - } while thisLoopChanged - } -} - -private func getWorkList(topLevelLoop: Loop, context: Context) -> Stack { - var tmp1 = Stack(context) - var tmp2 = Stack(context) - var workList = Stack(context) - defer { - tmp1.deinitialize() - tmp2.deinitialize() - } - - tmp1.push(topLevelLoop) - - while !tmp1.isEmpty || !tmp2.isEmpty { - while let loop = tmp2.pop() { - workList.push(loop) - } - - while let loop = tmp1.pop() { - tmp2.push(loop) - tmp1.append(contentsOf: loop.innerLoops) - } - } - - return workList -} - private func analyzeLoop( loop: Loop, context: FunctionPassContext @@ -183,9 +181,8 @@ private func analyzeLoop( defer { blockSideEffects.deinitialize() } for inst in bb.instructions { - if hasOwnershipOperandsOrResults(inst) { - checkSideEffects( - of: inst, + if inst.hasOwnershipOperandsOrResults { + inst.analyzeSideEffects( analyzedInstructions: &analyzedInstructions, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts @@ -213,16 +210,14 @@ private func analyzeLoop( } analyzedInstructions.append(storeInst) movableInstructions.loadsAndStores.append(storeInst) - checkSideEffects( - of: storeInst, + storeInst.analyzeSideEffects( analyzedInstructions: &analyzedInstructions, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts ) case let beginAccessInst as BeginAccessInst: analyzedInstructions.append(beginAccessInst) - checkSideEffects( - of: beginAccessInst, + beginAccessInst.analyzeSideEffects( analyzedInstructions: &analyzedInstructions, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts @@ -231,22 +226,19 @@ private func analyzeLoop( movableInstructions.specialHoist.append(refElementAddrInst) case let condFailInst as CondFailInst: movableInstructions.hoistUp.append(condFailInst) - checkSideEffects( - of: condFailInst, + condFailInst.analyzeSideEffects( analyzedInstructions: &analyzedInstructions, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts ) case let applyInst as ApplyInst: - if isSafeReadOnlyApply( - applyInst: applyInst, + if applyInst.isSafeReadOnlyApply( calleeAnalysis: context.calleeAnalysis ) { analyzedInstructions.append(applyInst) } else if let callee = applyInst.referencedFunction, callee.isGlobalInitFunction, - !mayConflictWithGlobalInit( - globalInitCall: applyInst, + !applyInst.globalInitMayConflictWith( sideEffects: blockSideEffects, aliasAnalysis: context.aliasAnalysis ) { @@ -261,8 +253,7 @@ private func analyzeLoop( case let builtinInst as BuiltinInst: switch builtinInst.id { case .Once, .OnceWithContext: - if !mayConflictWithGlobalInit( - globalInitCall: builtinInst, + if !builtinInst.globalInitMayConflictWith( sideEffects: blockSideEffects, aliasAnalysis: context.aliasAnalysis ) { @@ -273,16 +264,14 @@ private func analyzeLoop( default: break } - checkSideEffects( - of: inst, + inst.analyzeSideEffects( analyzedInstructions: &analyzedInstructions, blockSideEffects: &blockSideEffects, hasOtherMemReadingInsts: &hasOtherMemReadingInsts ) - if canHoistUpDefault( - inst: inst, - loop: loop, + if inst.canBeHoisted( + outOf: loop, context: context ) { movableInstructions.hoistUp.append(inst) @@ -294,8 +283,7 @@ private func analyzeLoop( // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. if analyzedInstructions.readOnlyAppliesCount * analyzedInstructions.loopSideEffectCount < 8000 { for readOnlyApply in analyzedInstructions.readOnlyApplies { - if !mayWriteTo( - readOnlyApply, + if !readOnlyApply.mayWriteTo( sideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis, calleeAnalysis: context.calleeAnalysis @@ -308,8 +296,7 @@ private func analyzeLoop( // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. if analyzedInstructions.loadsCount * analyzedInstructions.loopSideEffectCount < 8000 { for load in analyzedInstructions.loads { - if !mayWriteTo( - load, + if !load.mayWriteTo( sideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis ) { @@ -320,8 +307,7 @@ private func analyzeLoop( if !analyzedInstructions.globalInitCalls.isEmpty { for globalInitCall in analyzedInstructions.globalInitCalls { - if !mayConflictWithGlobalInit( - globalInitCall: globalInitCall, + if !globalInitCall.globalInitMayConflictWith( preheader: preheader, sideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis, @@ -337,20 +323,18 @@ private func analyzeLoop( let accessPath = storeInst.destination.accessPath if accessPath.isLoopInvariant(loop: loop), - isOnlyLoadedAndStored( - accessPath: accessPath, + accessPath.isOnlyLoadedAndStored( storeAddr: storeInst.destination, analyzedInstructions: analyzedInstructions, aliasAnalysis: context.aliasAnalysis - ), - !movableInstructions.loadAndStoreAddrs.contains(accessPath), - splitLoads( - analyzedInstructions: &analyzedInstructions, - storeAddr: storeInst.destination, - movableInstructions: &movableInstructions, - accessPath: accessPath, - context: context - ) { + ), + !movableInstructions.loadAndStoreAddrs.contains(accessPath), + movableInstructions.splitLoads( + analyzedInstructions: &analyzedInstructions, + storeAddr: storeInst.destination, + accessPath: accessPath, + context: context + ) { movableInstructions.loadAndStoreAddrs.append(accessPath) } } @@ -362,8 +346,7 @@ private func analyzeLoop( for fixLifetime in analyzedInstructions.fixLifetimes { guard fixLifetime.operand.value.type.isAddress else { continue } - if sideEffectsMayRelease || !mayWriteTo( - fixLifetime, + if sideEffectsMayRelease || !fixLifetime.mayWriteTo( sideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis ) { @@ -373,12 +356,8 @@ private func analyzeLoop( } for beginAccessInst in analyzedInstructions.beginAccesses { - if handledEndAccess( - beginAccessInst: beginAccessInst, - loop: loop, - context: context - ) && analyzeBeginAccess( - beginAccessInst: beginAccessInst, + if beginAccessInst.canBeHoisted( + outOf: loop, analyzedInstructions: analyzedInstructions, aliasAnalysis: context.aliasAnalysis, domTree: context.dominatorTree @@ -395,13 +374,8 @@ private func optimizeLoop( movableInstructions: inout MovableInstructions, context: FunctionPassContext ) -> Bool { - guard loop.preheader != nil else { - return false - } - - if hoistAllLoadsAndStores( - loop: loop, - movableInstructions: &movableInstructions, + if movableInstructions.hoistAllLoadsAndStores( + outOf: loop, context: context ) { return true @@ -409,860 +383,724 @@ private func optimizeLoop( var changed = false - changed = - hoistInstructions( + changed = movableInstructions.hoistInstructions( loop: loop, - hoistUp: movableInstructions.hoistUp, context: context ) || changed - changed = - sinkInstructions( + changed = movableInstructions.sinkInstructions( loop: loop, - sinkDown: movableInstructions.sinkDown, context: context ) || changed - changed = - hoistSpecialInstruction( + changed = movableInstructions.hoistWithSinkScopedInstructions( loop: loop, - specialInsts: movableInstructions.specialHoist, context: context ) || changed return changed } -func hoistAllLoadsAndStores( - loop: Loop, - movableInstructions: inout MovableInstructions, - context: FunctionPassContext -) -> Bool { - var changed = false - for accessPath in movableInstructions.loadAndStoreAddrs { - changed = hoistLoadsAndStores( - loop: loop, - loadsAndStores: &movableInstructions.loadsAndStores, - accessPath: accessPath, - context: context - ) || changed - } - - return changed -} +extension MovableInstructions { + mutating func splitLoads( + analyzedInstructions: inout AnalyzedInstructions, + storeAddr: Value, + accessPath: AccessPath, + context: FunctionPassContext + ) -> Bool { + var tmpLoads = Stack(context) + defer { + analyzedInstructions.append(newLoads: tmpLoads) + tmpLoads.deinitialize() + } + var splitCounter = 0 -func hoistLoadsAndStores( - loop: Loop, - loadsAndStores: inout [Instruction], - accessPath: AccessPath, - context: FunctionPassContext -) -> Bool { - let exitingAndLatchBlocks = loop.exitingAndLatchBlocks - - guard storesCommonlyDominateLoopExits( - loop: loop, - accessPath: accessPath, - exitingBlocks: exitingAndLatchBlocks, - context: context - ) else { - return false - } - - for exitingOrLatchBlock in exitingAndLatchBlocks { - for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !loop.basicBlocks.contains(succesor) { - _ = context.loopTree.splitCriticalEdge( - basicBlock: exitingOrLatchBlock.terminator.parentBlock, - edgeIndex: index, - domTree: context.dominatorTree - ) + while let loadInst = analyzedInstructions.popLoad() { + guard splitCounter <= 6 else { + tmpLoads.push(loadInst) + return false + } + + guard !loadInst.isDeleted, + loadInst.mayRead(fromAddress: storeAddr, context.aliasAnalysis), + !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) + else { + tmpLoads.push(loadInst) + continue + } + + let loadAccessPath = loadInst.operand.value.accessPath + guard !accessPath.isEqualOrContains(loadAccessPath) else { + tmpLoads.push(loadInst) + continue + } + + if let splitLoads = loadInst.trySplit(alongPath: accessPath.projectionPath, context) { + splitCounter += splitLoads.count + loadsAndStores.replace([loadInst], with: splitLoads) + tmpLoads.append(contentsOf: splitLoads) + } } + + return true } - guard let preheader = loop.preheader else { - return false - } - - var changed = false - let builder = Builder(before: preheader.terminator, context) - var ssaUpdater: SSAUpdater? - var storeAddr: Value? - - for case let storeInst as StoreInst in loadsAndStores where isStore(storeInst, thatAccesses: accessPath) { - if let srcLoadInst = storeInst.source as? LoadInst, - isLoad(srcLoadInst, withinAccess: accessPath) { - return changed - } - - if storeAddr == nil { - storeAddr = storeInst.destination - ssaUpdater = SSAUpdater( - function: storeAddr!.parentFunction, - type: storeAddr!.type.objectType, - ownership: storeAddr!.ownership, - context - ) - } else if storeInst.destination.type != storeAddr!.type { - return changed + mutating func hoistAllLoadsAndStores( + outOf loop: Loop, + context: FunctionPassContext + ) -> Bool { + var changed = false + for accessPath in loadAndStoreAddrs { + changed = hoistAndSinkLoadsAndStores( + outOf: loop, + accessPath: accessPath, + context: context + ) || changed } - ssaUpdater?.addAvailableValue(storeInst.source, in: storeInst.parentBlock) + return changed } - guard let storeAddr, var ssaUpdater else { + mutating func hoistInstructions( + loop: Loop, + context: FunctionPassContext + ) -> Bool { + let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(domTree: context.dominatorTree) + var changed = false + + for bb in dominatingBlocks { + for inst in bb.instructions where hoistUp.contains(inst) { + changed = inst.hoist( + outOf: loop, + context: context + ) || changed + } + } + return changed } - - var cloner = Cloner(cloneBefore: preheader.terminator, context) - defer { cloner.deinitialize() } - - guard let initialAddr = (cloner.cloneUseDefChain(addr: storeAddr) { srcAddr in - srcAddr.parentBlock.dominates(preheader, context.dominatorTree) - }) else { + + mutating func sinkInstructions( + loop: Loop, + context: FunctionPassContext + ) -> Bool { + let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(domTree: context.dominatorTree) + var changed = false + + for inst in sinkDown where dominatingBlocks.contains(inst.parentBlock) { + changed = inst.sink( + outOf: loop, + context: context + ) || changed + } + return changed } - - let ownership: LoadInst.LoadOwnership = preheader.terminator.parentFunction.hasOwnership ? .trivial : .unqualified - - let initialLoad = builder.createLoad(fromAddress: initialAddr, ownership: ownership) - ssaUpdater.addAvailableValue(initialLoad, in: preheader) - - var currentBlock: BasicBlock? - var currentVal: Value? - - // This loop depends on loadsAndStores being - // in order the instructions appear in blocks! - for inst in loadsAndStores { - let block = inst.parentBlock + + mutating func hoistWithSinkScopedInstructions( + loop: Loop, + context: FunctionPassContext + ) -> Bool { + var changed = false + + for specialInst in specialHoist { + if specialInst is BeginAccessInst && loop.hasNoExitBlocks { + // TODO: We should move this as a precondition out of the loop once we remove RefElementAddrInst from here. + continue + } + + guard + specialInst.hoist(outOf: loop, context: context) + else { + continue + } + + // TODO: This should probably be moved to the hoistUp collection. We should keep special hoist to only BeginAccessInst. + if let beginAccessInst = specialInst as? BeginAccessInst { + for endAccess in beginAccessInst.endAccessInstructions { + endAccess.sink(outOf: loop, context: context) + } + } + + changed = true + } + + return changed + } + + private mutating func hoistAndSinkLoadsAndStores( + outOf loop: Loop, + accessPath: AccessPath, + context: FunctionPassContext + ) -> Bool { + let exitingAndLatchBlocks = loop.exitingAndLatchBlocks - if block != currentBlock { - currentBlock = block - currentVal = nil + guard loop.storesCommonlyDominateExits( + accessPath: accessPath, + context: context + ) else { + return false } - if let storeInst = inst as? StoreInst, isStore(storeInst, thatAccesses: accessPath) { - currentVal = storeInst.source - loadsAndStores.removeAll(where: { $0 == storeInst }) - context.erase(instruction: storeInst) - changed = true - continue + for exitingOrLatchBlock in exitingAndLatchBlocks { + for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !loop.basicBlocks.contains(succesor) { + _ = context.loopTree.splitCriticalEdge( + basicBlock: exitingOrLatchBlock.terminator.parentBlock, + edgeIndex: index, + domTree: context.dominatorTree + ) + } + } + + guard let preheader = loop.preheader else { + return false } - guard let loadInst = inst as? LoadInst, - isLoad(loadInst, withinAccess: accessPath) else { - continue + var changed = false + let builder = Builder(before: preheader.terminator, context) + var ssaUpdater: SSAUpdater? + var storeAddr: Value? + + for case let storeInst as StoreInst in loadsAndStores where storeInst.accesses(accessPath) { + if let srcLoadInst = storeInst.source as? LoadInst, + srcLoadInst.accesses(accessPath) { + return changed + } + + if storeAddr == nil { + storeAddr = storeInst.destination + ssaUpdater = SSAUpdater( + function: storeAddr!.parentFunction, + type: storeAddr!.type.objectType, + ownership: storeAddr!.ownership, + context + ) + } else if storeInst.destination.type != storeAddr!.type { + return changed + } + + ssaUpdater?.addAvailableValue(storeInst.source, in: storeInst.parentBlock) } - let rootVal = currentVal ?? ssaUpdater.getValue(inMiddleOf: block) - - guard let projectedValue = projectLoadValue( - addr: loadInst.operand.value, - accessPath: loadInst.operand.value.accessPath, - rootVal: rootVal, - rootAccessPath: accessPath, - beforeInst: loadInst, - context: context - ) else { - continue + guard let storeAddr, var ssaUpdater else { + return changed } - loadInst.replace(with: projectedValue, context) - loadsAndStores.removeAll(where: { $0 == loadInst }) - changed = true - } - - for exitingOrLatchBlock in exitingAndLatchBlocks { - for succesor in exitingOrLatchBlock.successors where !loop.basicBlocks.contains(succesor) { - guard let firstInst = succesor.instructions.first else { + var cloner = Cloner(cloneBefore: preheader.terminator, context) + defer { cloner.deinitialize() } + + guard let initialAddr = (cloner.cloneUseDefChain(addr: storeAddr) { srcAddr in + srcAddr.parentBlock.dominates(preheader, context.dominatorTree) + }) else { + return changed + } + + let ownership: LoadInst.LoadOwnership = preheader.terminator.parentFunction.hasOwnership ? .trivial : .unqualified + + let initialLoad = builder.createLoad(fromAddress: initialAddr, ownership: ownership) + ssaUpdater.addAvailableValue(initialLoad, in: preheader) + + var currentBlock: BasicBlock? + var currentVal: Value? + + // This loop depends on loadsAndStores being + // in order the instructions appear in blocks! + for inst in loadsAndStores { + let block = inst.parentBlock + + if block != currentBlock { + currentBlock = block + currentVal = nil + } + + if let storeInst = inst as? StoreInst, storeInst.accesses(accessPath) { + currentVal = storeInst.source + loadsAndStores.removeAll(where: { $0 == storeInst }) + context.erase(instruction: storeInst) + changed = true continue } - let builder = Builder(before: firstInst, context) + guard let loadInst = inst as? LoadInst, + loadInst.accesses(accessPath) else { + continue + } - let ownership: StoreInst.StoreOwnership = firstInst.parentFunction.hasOwnership ? .trivial : .unqualified - builder.createStore( - source: ssaUpdater.getValue(inMiddleOf: succesor), - destination: initialAddr, - ownership: ownership - ) + let rootVal = currentVal ?? ssaUpdater.getValue(inMiddleOf: block) + + guard loadInst.replaceLoadWithProjection( + rootVal: rootVal, + rootAccessPath: accessPath, + beforeInst: loadInst, + context: context + ) else { + continue + } + + loadsAndStores.removeAll(where: { $0 == loadInst }) changed = true } + + for exitingOrLatchBlock in exitingAndLatchBlocks { + for succesor in exitingOrLatchBlock.successors where !loop.basicBlocks.contains(succesor) { + guard let firstInst = succesor.instructions.first else { + continue + } + + let builder = Builder(before: firstInst, context) + + let ownership: StoreInst.StoreOwnership = firstInst.parentFunction.hasOwnership ? .trivial : .unqualified + builder.createStore( + source: ssaUpdater.getValue(inMiddleOf: succesor), + destination: initialAddr, + ownership: ownership + ) + changed = true + } + } + + if initialLoad.isTriviallyDead { + context.erase(instruction: initialLoad) + } + + return changed } - - if initialLoad.isTriviallyDead { - context.erase(instruction: initialLoad) - } - - return changed } -func projectLoadValue( - addr: Value, - accessPath: AccessPath, - rootVal: Value, - rootAccessPath: AccessPath, - beforeInst: Instruction, - context: FunctionPassContext -) -> Value? { - guard accessPath != rootAccessPath else { - return rootVal - } - - guard let projectionPath = rootAccessPath.getProjection(to: accessPath) else { - return nil - } - - let builder = Builder(before: beforeInst, context) - - var currPath = projectionPath - var currVal = rootVal - - while !currPath.isEmpty { - let (kind, index, remainderPath) = currPath.pop() +extension Loop { + fileprivate func storesCommonlyDominateExits( + accessPath: AccessPath, + context: FunctionPassContext + ) -> Bool { + var stores = BasicBlockSet(context) + var storesNotAlive = BasicBlockSet(context) + var uses = Stack(context) + defer { + stores.deinitialize() + storesNotAlive.deinitialize() + uses.deinitialize() + } - switch kind { - case .structField: - currVal = builder.createStructExtract(struct: currVal, fieldIndex: index) - case .tupleField: - currVal = builder.createTupleExtract(tuple: currVal, elementIndex: index) - default: - return nil + for use in header.parentFunction.instructions.flatMap(\.operands) where use.value.accessPath == accessPath { + if let user = use.instruction as? StoreInst { + stores.insert(user.parentBlock) + } + } + + guard !stores.contains(header) else { + return true + } + + if let preheader = preheader, + stores.contains(preheader) { + return true + } + + storesNotAlive.insert(header) + + var changed = false + repeat { + changed = false + for block in basicBlocks where !storesNotAlive.contains(block) && !stores.contains(block) && block.predecessors.contains(where: { storesNotAlive.contains($0) }) { + storesNotAlive.insert(block) + changed = true + } + } while changed + + for exitingBlock in exitingBlocks where !exitingBlock.successors.contains(where: { $0.terminator is UnreachableInst }) && storesNotAlive.contains(exitingBlock) { + return false } - currPath = remainderPath + return true } - - return currVal } -func storesCommonlyDominateLoopExits( - loop: Loop, - accessPath: AccessPath, - exitingBlocks: some Sequence, - context: FunctionPassContext -) -> Bool { - var stores = BasicBlockSet(context) - var storesNotAlive = BasicBlockSet(context) - var uses = Stack(context) - defer { - stores.deinitialize() - storesNotAlive.deinitialize() - uses.deinitialize() - } - - for use in loop.header.parentFunction.instructions.flatMap(\.operands) where use.value.accessPath == accessPath { - if let user = use.instruction as? StoreInst { - stores.insert(user.parentBlock) +extension Instruction { + fileprivate func analyzeSideEffects( + analyzedInstructions: inout AnalyzedInstructions, + blockSideEffects: inout Stack, + hasOtherMemReadingInsts: inout Bool + ) { + if mayHaveSideEffects { + analyzedInstructions.appendSideEffect(self) + blockSideEffects.append(self) + } else if mayReadFromMemory { + hasOtherMemReadingInsts = true } } - guard !stores.contains(loop.header) else { - return true - } - - if let preheader = loop.preheader, - stores.contains(preheader) { - return true + fileprivate var hasOwnershipOperandsOrResults: Bool { + guard parentFunction.hasOwnership else { return false } + + return results.contains(where: { $0.ownership != .none }) + || operands.contains(where: { $0.value.ownership != .none }) } - storesNotAlive.insert(loop.header) - - var changed = false - repeat { - changed = false - for block in loop.basicBlocks where !storesNotAlive.contains(block) && !stores.contains(block) && block.predecessors.contains(where: { storesNotAlive.contains($0) }) { - storesNotAlive.insert(block) - changed = true + fileprivate func canBeHoisted( + outOf loop: Loop, + context: FunctionPassContext + ) -> Bool { + guard let preheader = loop.preheader else { + return false } - } while changed - - for exitingBlock in exitingBlocks where !exitingBlock.successors.contains(where: { $0.terminator is UnreachableInst }) && storesNotAlive.contains(exitingBlock) { - return false - } - - return true -} - -func hoistInstructions( - loop: Loop, - hoistUp: [Instruction], - context: FunctionPassContext -) -> Bool { - guard let preheader = loop.preheader else { - return false - } - var dominatingBlocks = getDominatingBlocks(loop: loop, context: context) - defer { dominatingBlocks.deinitialize() } - var changed = false - - for bb in dominatingBlocks { - for inst in bb.instructions where hoistUp.contains(inst) { - changed = - hoistInstruction( - inst: inst, - preheader: preheader, - loop: loop, - context: context - ) || changed + if self is TermInst || self is Allocation || self is Deallocation { + return false } - } - - return changed -} - -private func hoistInstruction( - inst: Instruction, - preheader: BasicBlock, - loop: Loop, - context: FunctionPassContext -) -> Bool { - guard inst.operands.allSatisfy({ !loop.basicBlocks.contains($0.value.parentBlock) }) else { - return false - } - let terminator = preheader.terminator - if inst.canHoistArraySemanticsCall(to: terminator, context) { - inst.hoistArraySemanticsCall(before: terminator, context) - } else { - inst.move(before: terminator, context) - } - - return true -} - -private func sinkInstructions( - loop: Loop, - sinkDown: [Instruction], - context: FunctionPassContext -) -> Bool { - var dominatingBlocks = getDominatingBlocks(loop: loop, context: context) - defer { dominatingBlocks.deinitialize() } - var changed = false - - for inst in sinkDown where dominatingBlocks.contains(inst.parentBlock) { - changed = sinkInstruction( - loop: loop, - inst: inst, - context: context - ) || changed - } - - return changed -} - -private func sinkInstruction( - loop: Loop, - inst: Instruction, - context: FunctionPassContext -) -> Bool { - var isSingleExit = loop.isSingleExit - let exitBlocks = loop.exitBlocks - let exitingBlocks = loop.exitingBlocks - var newExitBlocks = Stack(context) - defer { newExitBlocks.deinitialize() } - var changed = false - - for exitingBlock in exitingBlocks { - for (succesorIndex, succesor) in exitingBlock.successors.enumerated().reversed() - where !newExitBlocks.contains(succesor) && exitBlocks.contains(succesor) { - - let outsideBlock = context.loopTree.splitCriticalEdge( - basicBlock: exitingBlock, - edgeIndex: succesorIndex, - domTree: context.dominatorTree - ) ?? succesor - - newExitBlocks.push(outsideBlock) - - if outsideBlock.instructions.contains(where: { inst.isIdenticalTo($0) }) { - isSingleExit = false - } else if isSingleExit, let firstInstruction = outsideBlock.instructions.first { - inst.move(before: firstInstruction, context) - } else if let firstInstruction = outsideBlock.instructions.first { - inst.copy(before: firstInstruction, context) - } else { - continue + switch getArraySemanticsCallKind() { + case .getCount, .getCapacity: + if canHoistArraySemanticsCall(to: preheader.terminator, context) { + return true } + case .arrayPropsIsNativeTypeChecked: + return false + default: + break + } - changed = true + if memoryEffects == .noEffects { + return true } - } - if changed && !isSingleExit { - context.erase(instruction: inst) + return false } - - return changed -} - -private func hoistSpecialInstruction( - loop: Loop, - specialInsts: [Instruction], - context: FunctionPassContext -) -> Bool { - guard let preheader = loop.preheader else { return false } - - var changed = false - - for specialInst in specialInsts { - if specialInst is BeginAccessInst && loop.hasNoExitBlocks { - // TODO: We should move this as a precondition out of the loop once we remove RefElementAddrInst from here. - continue + + @discardableResult + fileprivate func hoist( + outOf loop: Loop, + context: FunctionPassContext + ) -> Bool { + guard let preheader = loop.preheader, + operands.allSatisfy({ !loop.basicBlocks.contains($0.value.parentBlock) }) else { + return false } - guard - hoistInstruction( - inst: specialInst, - preheader: preheader, - loop: loop, - context: context - ) - else { - continue + let terminator = preheader.terminator + if canHoistArraySemanticsCall(to: terminator, context) { + hoistArraySemanticsCall(before: terminator, context) + } else { + move(before: terminator, context) } - // TODO: This should probably be moved to the hoistUp collection. We should keep special hoist to only BeginAccessInst. - if let beginAccessInst = specialInst as? BeginAccessInst { - var endAccesses = getEndAccesses(beginAccessInst: beginAccessInst, context: context) - defer { endAccesses.deinitialize() } + return true + } + + @discardableResult + fileprivate func sink( + outOf loop: Loop, + context: FunctionPassContext + ) -> Bool { + var isSingleExit = loop.isSingleExit + let exitBlocks = loop.exitBlocks + let exitingBlocks = loop.exitingBlocks + var newExitBlocks = Stack(context) + defer { newExitBlocks.deinitialize() } + var changed = false + + for exitingBlock in exitingBlocks { + for (succesorIndex, succesor) in exitingBlock.successors.enumerated().reversed() + where !newExitBlocks.contains(succesor) && exitBlocks.contains(succesor) { + + let outsideBlock = context.loopTree.splitCriticalEdge( + basicBlock: exitingBlock, + edgeIndex: succesorIndex, + domTree: context.dominatorTree + ) ?? succesor + + newExitBlocks.push(outsideBlock) + + if outsideBlock.instructions.contains(where: { isIdenticalTo($0) }) { + isSingleExit = false + } else if isSingleExit, let firstInstruction = outsideBlock.instructions.first { + move(before: firstInstruction, context) + } else if let firstInstruction = outsideBlock.instructions.first { + copy(before: firstInstruction, context) + } else { + continue + } - for endAccess in endAccesses { - _ = sinkInstruction(loop: loop, inst: endAccess, context: context) + changed = true } } - - changed = true - } - - return changed -} -private func getDominatingBlocks( - loop: Loop, - context: FunctionPassContext -) -> Stack { - var domBlocks = Stack(context) - - getDominatingBlocksHelper( - bb: loop.header, - exitingAndLatchBBs: loop.exitingAndLatchBlocks, - domBlocks: &domBlocks, - domTree: context.dominatorTree - ) - - return domBlocks -} + if changed && !isSingleExit { + context.erase(instruction: self) + } -private func getDominatingBlocksHelper( - bb: BasicBlock, - exitingAndLatchBBs: some Sequence, - domBlocks: inout Stack, - domTree: DominatorTree -) { - guard exitingAndLatchBBs.allSatisfy({ exitBlock in - return bb.dominates(exitBlock, domTree) - }) else { - return + return changed } - domBlocks.push(bb) - - for child in domTree.getChildren(of: bb) { - getDominatingBlocksHelper( - bb: child, - exitingAndLatchBBs: exitingAndLatchBBs, - domBlocks: &domBlocks, - domTree: domTree - ) - } -} - -/// Returns `true` if `inst` may have side effects. -private func checkSideEffects( - of inst: Instruction, - analyzedInstructions: inout AnalyzedInstructions, - blockSideEffects: inout Stack, - hasOtherMemReadingInsts: inout Bool -) { - if inst.mayHaveSideEffects { - analyzedInstructions.appendSideEffect(inst) - blockSideEffects.append(inst) - } else if inst.mayReadFromMemory { - hasOtherMemReadingInsts = true - } -} - -private func hasOwnershipOperandsOrResults(_ inst: Instruction) -> Bool { - guard inst.parentFunction.hasOwnership else { return false } - - return inst.results.contains(where: { $0.ownership != .none }) - || inst.operands.contains(where: { $0.value.ownership != .none }) -} - -private func isSafeReadOnlyApply( - applyInst: ApplyInst, - calleeAnalysis: CalleeAnalysis -) -> Bool { - guard applyInst.functionConvention.results.allSatisfy({ $0.convention == .unowned }) else { - return false - } - - if let callee = applyInst.referencedFunction, - callee.hasSemanticsAttribute("array.props.isNativeTypeChecked") { - return false - } - - return calleeAnalysis.getSideEffects(ofApply: applyInst).isOnlyReading -} - -private func canHoistUpDefault( - inst: Instruction, - loop: Loop, - context: FunctionPassContext -) -> Bool { - guard let preheader = loop.preheader else { - return false + private func globalInitMayConflictWith( + sideEffect: Instruction, + aliasAnalysis: AliasAnalysis + ) -> Bool { + switch sideEffect { + case let storeInst as StoreInst: + return mayReadOrWrite(address: storeInst.destinationOperand.value, aliasAnalysis) + case let loadInst as LoadInst: + return mayWrite(toAddress: loadInst.operand.value, aliasAnalysis) + case is CondFailInst: + return false + default: + return true + } } - - if inst is TermInst || inst is Allocation || inst is Deallocation { - return false + + fileprivate func globalInitMayConflictWith( + sideEffects: Stack, + aliasAnalysis: AliasAnalysis + ) -> Bool { + return sideEffects + .contains { sideEffect in + globalInitMayConflictWith( + sideEffect: sideEffect, + aliasAnalysis: aliasAnalysis + ) + } } - switch inst.getArraySemanticsCallKind() { - case .getCount, .getCapacity: - if inst.canHoistArraySemanticsCall(to: preheader.terminator, context) - { + fileprivate func globalInitMayConflictWith( + preheader: BasicBlock, + sideEffects: Stack, + aliasAnalysis: AliasAnalysis, + postDomTree: PostDominatorTree + ) -> Bool { + guard parentBlock.postDominates(preheader, postDomTree) else { return true } - case .arrayPropsIsNativeTypeChecked: - return false - default: - break - } - if inst.memoryEffects == .noEffects { - return true + return sideEffects + .contains { sideEffect in + parentBlock.strictlyPostDominates( + sideEffect.parentBlock, + postDomTree + ) && globalInitMayConflictWith( + sideEffect: sideEffect, + aliasAnalysis: aliasAnalysis + ) + } } - - return false } -private func mayWriteTo( - _ applyInst: ApplyInst, - sideEffects: Stack, - aliasAnalysis: AliasAnalysis, - calleeAnalysis: CalleeAnalysis -) -> Bool { - guard calleeAnalysis.getSideEffects(ofApply: applyInst).memory == .noEffects else { - return false - } - - for sideEffect in sideEffects { - switch sideEffect { - case let storeInst as StoreInst: - if storeInst.storeOwnership == .assign - && applyInst.mayRead(fromAddress: storeInst.destination, aliasAnalysis) - { - return true - } - case let copyAddrInst as CopyAddrInst: - if !copyAddrInst.isInitializationOfDestination - && applyInst.mayRead(fromAddress: copyAddrInst.destination, aliasAnalysis) - { - return true - } - case is ApplyInst, is BeginApplyInst, is TryApplyInst: - if !calleeAnalysis.getSideEffects(ofApply: applyInst).isOnlyReading { - return true - } - case is CondFailInst, is StrongRetainInst, is UnmanagedRetainValueInst, - is RetainValueInst, is StrongRetainUnownedInst, is FixLifetimeInst, - is KeyPathInst, is DeallocStackInst, is DeallocStackRefInst, - is DeallocRefInst: - break - default: - if sideEffect.mayWriteToMemory { - return true +extension UnaryInstruction { + fileprivate func mayWriteTo( + sideEffects: Stack, + aliasAnalysis: AliasAnalysis + ) -> Bool { + return sideEffects + .contains { sideEffect in + sideEffect.mayWrite(toAddress: operand.value, aliasAnalysis) } - } } - - return false } -private func mayWriteTo( - _ unaryInst: UnaryInstruction, - sideEffects: Stack, - aliasAnalysis: AliasAnalysis -) -> Bool { - return - sideEffects - .contains { sideEffect in - sideEffect.mayWrite(toAddress: unaryInst.operand.value, aliasAnalysis) +extension StoreInst { + fileprivate func accesses(_ accessPath: AccessPath) -> Bool { + guard self.storeOwnership != .initialize else { + return false } -} -private func handledEndAccess( - beginAccessInst: BeginAccessInst, - loop: Loop, - context: Context -) -> Bool { - var endAccesses = getEndAccesses(beginAccessInst: beginAccessInst, context: context) - defer { endAccesses.deinitialize() } - - return !endAccesses.isEmpty - && !endAccesses - .contains { user in - !loop.basicBlocks.contains(user.parentBlock) - } -} - -private func getEndAccesses( - beginAccessInst: BeginAccessInst, - context: Context -) -> Stack { - var endAccesses = Stack(context) - - endAccesses.append( - contentsOf: beginAccessInst.uses.compactMap { user in - user.instruction as? EndAccessInst - }) - - return endAccesses -} - -private func mayConflictWithGlobalInit( - globalInitCall: Instruction, - sideEffect: Instruction, - aliasAnalysis: AliasAnalysis -) -> Bool { - switch sideEffect { - case let storeInst as StoreInst: - return globalInitCall.mayReadOrWrite(address: storeInst.destinationOperand.value, aliasAnalysis) - case let loadInst as LoadInst: - return globalInitCall.mayWrite(toAddress: loadInst.operand.value, aliasAnalysis) - case is CondFailInst: - return false - default: - return true + return accessPath == self.destination.accessPath } } -private func mayConflictWithGlobalInit( - globalInitCall: Instruction, - sideEffects: Stack, - aliasAnalysis: AliasAnalysis -) -> Bool { - return sideEffects - .contains { sideEffect in - mayConflictWithGlobalInit( - globalInitCall: globalInitCall, - sideEffect: sideEffect, - aliasAnalysis: aliasAnalysis - ) +extension LoadInst { + fileprivate func replaceLoadWithProjection( + rootVal: Value, + rootAccessPath: AccessPath, + beforeInst: Instruction, + context: FunctionPassContext + ) -> Bool { + guard operand.value.accessPath != rootAccessPath else { + replace(with: rootVal, context) + return true } -} - -private func mayConflictWithGlobalInit( - globalInitCall: Instruction, - preheader: BasicBlock, - sideEffects: Stack, - aliasAnalysis: AliasAnalysis, - postDomTree: PostDominatorTree -) -> Bool { - guard globalInitCall.parentBlock.postDominates(preheader, postDomTree) else { + + guard let projectionPath = rootAccessPath.getProjection(to: operand.value.accessPath) else { + return false + } + + let builder = Builder(before: beforeInst, context) + + var currPath = projectionPath + var currVal = rootVal + + while !currPath.isEmpty { + let (kind, index, remainderPath) = currPath.pop() + + switch kind { + case .structField: + currVal = builder.createStructExtract(struct: currVal, fieldIndex: index) + case .tupleField: + currVal = builder.createTupleExtract(tuple: currVal, elementIndex: index) + default: + return false + } + + currPath = remainderPath + } + + replace(with: currVal, context) return true } - - return sideEffects - .contains { sideEffect in - globalInitCall.parentBlock.strictlyPostDominates( - sideEffect.parentBlock, - postDomTree - ) && mayConflictWithGlobalInit( - globalInitCall: globalInitCall, - sideEffect: sideEffect, - aliasAnalysis: aliasAnalysis - ) + + fileprivate func accesses( + _ accessPath: AccessPath + ) -> Bool { + return accessPath.getProjection(to: self.address.accessPath)?.isMaterializable ?? false + } + + fileprivate func overlaps( + accessPath: AccessPath + ) -> Bool { + guard self.loadOwnership != .take else { // TODO: handle LoadOwnershipQualifier::Take + return false } + + return accessPath.isEqualOrContains(self.operand.value.accessPath) || self.operand.value.accessPath.isEqualOrContains(accessPath) + } } -private func isOnlyLoadedAndStored( - accessPath: AccessPath, - storeAddr: Value, - analyzedInstructions: AnalyzedInstructions, - aliasAnalysis: AliasAnalysis -) -> Bool { - return !analyzedInstructions.loopSideEffects - .contains { sideEffect in - sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) - && !isStore(sideEffect, thatAccesses: accessPath) - && !isLoad(sideEffect, withinAccess: accessPath) - } && !analyzedInstructions.loads - .contains { loadInst in - loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) - && !isLoadWithAccessPath(loadInst, thatOverlapsAccess: accessPath) - } && !analyzedInstructions.stores - .contains { storeInst in - storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) - && !isStore(storeInst, thatAccesses: accessPath) +extension ApplyInst { + fileprivate func isSafeReadOnlyApply( + calleeAnalysis: CalleeAnalysis + ) -> Bool { + guard functionConvention.results.allSatisfy({ $0.convention == .unowned }) else { + return false } -} -private func isStore( - _ inst: Instruction, - thatAccesses accessPath: AccessPath -) -> Bool { - guard let storeInst = inst as? StoreInst else { - return false - } + if let callee = referencedFunction, + callee.hasSemanticsAttribute("array.props.isNativeTypeChecked") { + return false + } - // TODO: handle StoreOwnershipQualifier::Init - guard storeInst.storeOwnership != .initialize else { - return false + return calleeAnalysis.getSideEffects(ofApply: self).isOnlyReading } + + fileprivate func mayWriteTo( + sideEffects: Stack, + aliasAnalysis: AliasAnalysis, + calleeAnalysis: CalleeAnalysis + ) -> Bool { + guard calleeAnalysis.getSideEffects(ofApply: self).memory == .noEffects else { + return false + } - return accessPath == storeInst.destination.accessPath -} + for sideEffect in sideEffects { + switch sideEffect { + case let storeInst as StoreInst: + if storeInst.storeOwnership == .assign && + mayRead(fromAddress: storeInst.destination, aliasAnalysis) { + return true + } + case let copyAddrInst as CopyAddrInst: + if !copyAddrInst.isInitializationOfDestination && + mayRead(fromAddress: copyAddrInst.destination, aliasAnalysis) { + return true + } + case is ApplyInst, is BeginApplyInst, is TryApplyInst: + if !calleeAnalysis.getSideEffects(ofApply: self).isOnlyReading { + return true + } + case is CondFailInst, is StrongRetainInst, is UnmanagedRetainValueInst, + is RetainValueInst, is StrongRetainUnownedInst, is FixLifetimeInst, + is KeyPathInst, is DeallocStackInst, is DeallocStackRefInst, + is DeallocRefInst: + break + default: + if sideEffect.mayWriteToMemory { + return true + } + } + } -private func isLoad( - _ inst: Instruction, - withinAccess accessPath: AccessPath -) -> Bool { - guard let loadInst = inst as? LoadInst else { return false } - - return accessPath.getProjection(to: loadInst.address.accessPath)?.isMaterializable ?? false } -private func isLoadWithAccessPath( - _ inst: Instruction, - thatOverlapsAccess accessPath: AccessPath -) -> Bool { - guard let loadInst = inst as? LoadInst, - loadInst.loadOwnership != .take // TODO: handle LoadOwnershipQualifier::Take - else { - return false +extension BeginAccessInst { + private func isCoveredByScope( + otherInst: Instruction, + domTree: DominatorTree + ) -> Bool { + return self.parentBlock.strictlyDominates( + otherInst.parentBlock, + domTree + ) && self.endAccessInstructions + .allSatisfy { endAccessInst in + otherInst.parentBlock.dominates(endAccessInst.parentBlock, domTree) + } } - return accessPath.isEqualOrContains(loadInst.operand.value.accessPath) || loadInst.operand.value.accessPath.isEqualOrContains(accessPath) -} - -private func splitLoads( - analyzedInstructions: inout AnalyzedInstructions, - storeAddr: Value, - movableInstructions: inout MovableInstructions, - accessPath: AccessPath, - context: FunctionPassContext -) -> Bool { - var tmpLoads = Stack(context) - defer { tmpLoads.deinitialize() } - var splitCounter = 0 - - while let loadInst = analyzedInstructions.popLoad() { - guard splitCounter <= 6 else { - tmpLoads.push(loadInst) - analyzedInstructions.append(newLoads: tmpLoads) - return false - } - - guard !loadInst.isDeleted, - loadInst.mayRead(fromAddress: storeAddr, context.aliasAnalysis), - !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) - else { - tmpLoads.push(loadInst) - continue - } - - let loadAccessPath = loadInst.operand.value.accessPath - guard !accessPath.isEqualOrContains(loadAccessPath) else { - tmpLoads.push(loadInst) - continue - } - - if let splitLoads: [LoadInst] = loadInst.trySplit(alongPath: accessPath.projectionPath, context) { - splitCounter += splitLoads.count - movableInstructions.loadsAndStores.replace([loadInst], with: splitLoads) - tmpLoads.append(contentsOf: splitLoads) - } + private func handlesAllEndAccesses( + loop: Loop + ) -> Bool { + return !endAccessInstructions.isEmpty && !endAccessInstructions + .contains { user in + !loop.basicBlocks.contains(user.parentBlock) + } } - analyzedInstructions.append(newLoads: tmpLoads) - - return true -} - -private func analyzeBeginAccess( - beginAccessInst: BeginAccessInst, - analyzedInstructions: AnalyzedInstructions, - aliasAnalysis: AliasAnalysis, - domTree: DominatorTree -) -> Bool { - let areBeginAccessesSafe = analyzedInstructions.beginAccesses - .allSatisfy { otherBeginAccessInst in - guard beginAccessInst != otherBeginAccessInst else { return true } - - return beginAccessInst.accessPath.isDistinct(from: otherBeginAccessInst.accessPath) - } - - guard areBeginAccessesSafe else { return false } - - for fullApplyInst in analyzedInstructions.fullApplies { - guard beginAccessInst.mayWriteToMemory - ? fullApplyInst.mayReadOrWrite( - address: beginAccessInst.address, - aliasAnalysis - ) : fullApplyInst.mayWrite( - toAddress: beginAccessInst.address, - aliasAnalysis - ) else { - continue - } - - if !isCoveredByScope( - beginAccessInst: beginAccessInst, - otherInst: fullApplyInst, - domTree: domTree - ) { + fileprivate func canBeHoisted( + outOf loop: Loop, + analyzedInstructions: AnalyzedInstructions, + aliasAnalysis: AliasAnalysis, + domTree: DominatorTree + ) -> Bool { + guard handlesAllEndAccesses(loop: loop) else { return false } - } + + let areBeginAccessesSafe = analyzedInstructions.beginAccesses + .allSatisfy { otherBeginAccessInst in + guard self != otherBeginAccessInst else { return true } + + return self.accessPath.isDistinct(from: otherBeginAccessInst.accessPath) + } - switch beginAccessInst.accessPath.base { - case .class, .global: - for sideEffect in analyzedInstructions.loopSideEffects { - guard sideEffect.mayRelease else { + guard areBeginAccessesSafe else { return false } + + for fullApplyInst in analyzedInstructions.fullApplies { + guard mayWriteToMemory + ? fullApplyInst.mayReadOrWrite( + address: address, + aliasAnalysis + ) : fullApplyInst.mayWrite( + toAddress: address, + aliasAnalysis + ) else { continue } if !isCoveredByScope( - beginAccessInst: beginAccessInst, - otherInst: sideEffect, + otherInst: fullApplyInst, domTree: domTree ) { return false } } - return true - default: - return true - } -} + switch accessPath.base { + case .class, .global: + for sideEffect in analyzedInstructions.loopSideEffects { + guard sideEffect.mayRelease else { + continue + } -private func isCoveredByScope( - beginAccessInst: BeginAccessInst, - otherInst: Instruction, - domTree: DominatorTree -) -> Bool { - return beginAccessInst.parentBlock.strictlyDominates( - otherInst.parentBlock, - domTree - ) && beginAccessInst.endAccessInstructions - .allSatisfy { endAccessInst in - otherInst.parentBlock.dominates(endAccessInst.parentBlock, domTree) + if !isCoveredByScope( + otherInst: sideEffect, + domTree: domTree + ) { + return false + } + } + + return true + default: + return true } + } } extension AccessPath { @@ -1288,4 +1126,36 @@ extension AccessPath { return projectionPath.isConstant } + + fileprivate func isOnlyLoadedAndStored( + storeAddr: Value, + analyzedInstructions: AnalyzedInstructions, + aliasAnalysis: AliasAnalysis + ) -> Bool { + return !analyzedInstructions.loopSideEffects + .contains { sideEffect in + let accessesPath: Bool + switch sideEffect { + case let storeInst as StoreInst: + accessesPath = storeInst.accesses(self) + case let loadInst as LoadInst: + accessesPath = loadInst.accesses(self) + default: + accessesPath = false + } + + return !accessesPath && sideEffect.mayReadOrWrite( + address: storeAddr, + aliasAnalysis + ) + } && !analyzedInstructions.loads + .contains { loadInst in + loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) + && !loadInst.overlaps(accessPath: self) + } && !analyzedInstructions.stores + .contains { storeInst in + storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) + && !storeInst.accesses(self) + } + } } From cf13e6fd25a48a6e4b60543f8dfe433141917ef6 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Thu, 31 Jul 2025 15:25:19 +0100 Subject: [PATCH 18/44] Add removal of duplicates when hoisting instructions. Removes the necessity of running cse sometimes after the pass. --- .../LoopInvariantCodeMotion.swift | 12 +- test/SILOptimizer/licm.sil | 154 ++++-------------- 2 files changed, 46 insertions(+), 120 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index e9425efbd43fc..8abef6773158d 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -321,7 +321,6 @@ private func analyzeLoop( if !hasOtherMemReadingInsts { for storeInst in analyzedInstructions.stores { let accessPath = storeInst.destination.accessPath - if accessPath.isLoopInvariant(loop: loop), accessPath.isOnlyLoadedAndStored( storeAddr: storeInst.destination, @@ -782,6 +781,17 @@ extension Instruction { } else { move(before: terminator, context) } + + if let singleValueInst = self as? SingleValueInstruction, + let identicalInst = (preheader.instructions.first { otherInst in + return singleValueInst != otherInst && singleValueInst.isIdenticalTo(otherInst) + }) { + guard let identicalSingleValueInst = identicalInst as? SingleValueInstruction else { + return true + } + + singleValueInst.replace(with: identicalSingleValueInst, context) + } return true } diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index ce799567c8dff..392dc98850c99 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -944,47 +944,10 @@ bb5: return %99 : $() } -// Test load splitting with a loop-invariant stored value. `struct_element_addr` will be -// hoisted and will be possible to merge by subsequently running CSE pass. -// -// CHECK-LABEL: sil shared @testLoadSplitBeforeCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { -// CHECK: struct_element_addr -// CHECK: struct_element_addr -// CHECK: struct_element_addr -// CHECK: br bb1 -// CHECK: bb1: -// CHECK: load -// CHECK: load -// CHECK: store -// CHECK: load -sil shared @testLoadSplitBeforeCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { -bb0(%0 : $Int64, %1 : $Builtin.RawPointer): - %outerAddr1 = pointer_to_address %1 : $Builtin.RawPointer to $*Index - %middleAddr1 = struct_element_addr %outerAddr1 : $*Index, #Index.value - br bb1 - -bb1: - %val1 = load %outerAddr1 : $*Index - %val2 = load %middleAddr1 : $*Int64 - %outerAddr2 = pointer_to_address %1 : $Builtin.RawPointer to $*Index - %middleAddr2 = struct_element_addr %outerAddr2 : $*Index, #Index.value - store %0 to %middleAddr2 : $*Int64 - %innerAddr1 = struct_element_addr %middleAddr1 : $*Int64, #Int64._value - %val3 = load %innerAddr1 : $*Builtin.Int64 - cond_br undef, bb2, bb3 - -bb2: - br bb1 - -bb3: - %result = tuple (%val1 : $Index, %val2 : $Int64, %val3 : $Builtin.Int64) - return %result : $(Index, Int64, Builtin.Int64) -} - -// Test load splitting with a loop-invariant stored value after CSE. The loop +// Test load splitting with a loop-invariant. The loop // will be empty after combined load/store hoisting/sinking. // -// CHECK-LABEL: sil shared @testLoadSplitAfterCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +// CHECK-LABEL: sil shared @testLoadSplit : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { // CHECK: [[PRELOAD:%.*]] = load %{{.*}} : $*Int64 // CHECK: [[STOREDVAL:%.*]] = struct_extract %0 : $Int64, #Int64._value // CHECK: br bb1([[PRELOAD]] : $Int64) @@ -996,48 +959,49 @@ bb3: // CHECK: bb3: // CHECK-NEXT: store %0 to %{{.*}} : $*Int64 // CHECK-NEXT: tuple ([[OUTERVAL]] : $Index, [[PHI]] : $Int64, [[STOREDVAL]] : $Builtin.Int64) -// CHECK-LABEL: } // end sil function 'testLoadSplitAfterCSE' -sil shared @testLoadSplitAfterCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +// CHECK-LABEL: } // end sil function 'testLoadSplit' +sil shared @testLoadSplit : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { bb0(%0 : $Int64, %1 : $Builtin.RawPointer): - %outerAddr = pointer_to_address %1 to $*Index - %middleAddr = struct_element_addr %outerAddr, #Index.value - %innerAddr = struct_element_addr %middleAddr, #Int64._value + %outerAddr1 = pointer_to_address %1 : $Builtin.RawPointer to $*Index + %middleAddr1 = struct_element_addr %outerAddr1 : $*Index, #Index.value br bb1 bb1: - %val1 = load %outerAddr - %val2 = load %middleAddr - store %0 to %middleAddr - %val3 = load %innerAddr + %val1 = load %outerAddr1 : $*Index + %val2 = load %middleAddr1 : $*Int64 + %outerAddr2 = pointer_to_address %1 : $Builtin.RawPointer to $*Index + %middleAddr2 = struct_element_addr %outerAddr2 : $*Index, #Index.value + store %0 to %middleAddr2 : $*Int64 + %innerAddr1 = struct_element_addr %middleAddr1 : $*Int64, #Int64._value + %val3 = load %innerAddr1 : $*Builtin.Int64 cond_br undef, bb2, bb3 bb2: br bb1 bb3: - %result = tuple (%val1, %val2, %val3) - return %result -} // end sil function 'testLoadSplitAfterCSE' + %result = tuple (%val1 : $Index, %val2 : $Int64, %val3 : $Builtin.Int64) + return %result : $(Index, Int64, Builtin.Int64) +} // end sil function 'testLoadSplit' // Test load splitting with a loop-varying stored value. -// CHECK-LABEL: sil shared @testLoadSplitPhiBeforeCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { -// CHECK: pointer_to_address -// CHECK: struct_element_addr -// CHECK: struct_element_addr -// CHECK: integer_literal -// CHECK: integer_literal -// CHECK: pointer_to_address -// CHECK: struct_element_addr -// CHECK: br bb1 -// CHECK: bb1: -// CHECK: load -// CHECK: load -// CHECK: load +// CHECK-LABEL: sil shared @testLoadSplitPhi : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +// CHECK: [[PRELOAD:%.*]] = load %{{.*}} : $*Int64 +// CHECK: br bb1(%{{.*}} : $Int64) +// CHECK: bb1([[PHI:%.*]] : $Int64): +// CHECK-NEXT: [[OUTERVAL:%.*]] = struct $Index ([[PHI]] : $Int64) +// CHECK-NEXT: [[EXTRACT:%.*]] = struct_extract [[PHI]] : $Int64, #Int64._value +// CHECK-NEXT: builtin "uadd_with_overflow_Int32"([[EXTRACT]] : $Builtin.Int64 +// CHECK-NEXT: tuple_extract +// CHECK-NEXT: [[ADD:%.*]] = struct $Int64 // CHECK-NEXT: cond_br undef, bb2, bb3 // CHECK: bb2: -// CHECK-NEXT: br bb1 -// CHECK-LABEL: } // end sil function 'testLoadSplitPhiBeforeCSE' -sil shared @testLoadSplitPhiBeforeCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { +// CHECK-NEXT: br bb1([[ADD]] : $Int64) +// CHECK: bb3: +// CHECK-NEXT: store [[ADD]] to %{{.*}} : $*Int64 +// CHECK-NEXT: tuple ([[OUTERVAL]] : $Index, [[ADD]] : $Int64, [[EXTRACT]] : $Builtin.Int64) +// CHECK-LABEL: } // end sil function 'testLoadSplitPhi' +sil shared @testLoadSplitPhi : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { bb0(%0 : $Int64, %1 : $Builtin.RawPointer): %outerAddr1 = pointer_to_address %1 : $Builtin.RawPointer to $*Index %middleAddr1 = struct_element_addr %outerAddr1 : $*Index, #Index.value @@ -1064,52 +1028,7 @@ bb2: bb3: %result = tuple (%outerVal : $Index, %middleVal : $Int64, %innerVal : $Builtin.Int64) return %result : $(Index, Int64, Builtin.Int64) -} - -// Test load splitting with a loop-varying stored value. -// CHECK-LABEL: sil shared @testLoadSplitPhiAfterCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { -// CHECK: [[PRELOAD:%.*]] = load %{{.*}} : $*Int64 -// CHECK: br bb1(%{{.*}} : $Int64) -// CHECK: bb1([[PHI:%.*]] : $Int64): -// CHECK-NEXT: [[OUTERVAL:%.*]] = struct $Index ([[PHI]] : $Int64) -// CHECK-NEXT: [[EXTRACT:%.*]] = struct_extract [[PHI]] : $Int64, #Int64._value -// CHECK-NEXT: builtin "uadd_with_overflow_Int32"([[EXTRACT]] : $Builtin.Int64 -// CHECK-NEXT: tuple_extract -// CHECK-NEXT: [[ADD:%.*]] = struct $Int64 -// CHECK-NEXT: cond_br undef, bb2, bb3 -// CHECK: bb2: -// CHECK-NEXT: br bb1([[ADD]] : $Int64) -// CHECK: bb3: -// CHECK-NEXT: store [[ADD]] to %{{.*}} : $*Int64 -// CHECK-NEXT: tuple ([[OUTERVAL]] : $Index, [[ADD]] : $Int64, [[EXTRACT]] : $Builtin.Int64) -// CHECK-LABEL: } // end sil function 'testLoadSplitPhiAfterCSE' -sil shared @testLoadSplitPhiAfterCSE : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { -// %1 // user: %2 -bb0(%0 : $Int64, %1 : $Builtin.RawPointer): - %2 = pointer_to_address %1 to $*Index // users: %8, %3 - %3 = struct_element_addr %2, #Index.value // users: %13, %14, %4 - %4 = struct_element_addr %3, #Int64._value // user: %9 - %5 = integer_literal $Builtin.Int64, 1 // user: %10 - %6 = integer_literal $Builtin.Int1, 0 // user: %10 - br bb1 // id: %7 - -bb1: // Preds: bb2 bb0 - %8 = load %2 // user: %17 - %9 = load %4 // users: %17, %10 - %10 = builtin "uadd_with_overflow_Int32"(%9, %5, %6) : $(Builtin.Int64, Builtin.Int1) // user: %11 - %11 = tuple_extract %10, 0 // user: %12 - %12 = struct $Int64 (%11) // user: %13 - store %12 to %3 // id: %13 - %14 = load %3 // user: %17 - cond_br undef, bb2, bb3 // id: %15 - -bb2: // Preds: bb1 - br bb1 // id: %16 - -bb3: // Preds: bb1 - %17 = tuple (%8, %14, %9) // user: %18 - return %17 // id: %18 -} // end sil function 'testLoadSplitPhiAfterCSE' +} // end sil function 'testLoadSplitPhi' struct State { @_hasStorage var valueSet: (Int64, Int64, Int64) { get set } @@ -1128,8 +1047,7 @@ struct State { // CHECK: [[PRELOADADR:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 1 // CHECK: [[PRELOAD:%.*]] = load [[PRELOADADR]] : $*Int64 // ...Split element 0 -// CHECK: [[SPLIT0:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 0 -// CHECK: [[ELT0:%.*]] = load [[SPLIT0]] : $*Int64 +// CHECK: [[ELT0:%.*]] = load [[HOISTADR]] : $*Int64 // ...Split element 2 // CHECK: [[SPLIT2:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 2 // CHECK: [[ELT2:%.*]] = load [[SPLIT2]] : $*Int64 @@ -1137,8 +1055,7 @@ struct State { // CHECK: [[SINGLEADR:%.*]] = struct_element_addr %{{.*}} : $*State, #State.singleValue // CHECK: [[SINGLEVAL:%.*]] = load [[SINGLEADR]] : $*Int64 // ...Hoisted element 0 -// CHECK: [[HOISTLOAD:%.*]] = load [[HOISTADR]] : $*Int64 -// CHECK: [[HOISTVAL:%.*]] = struct_extract [[HOISTLOAD]] : $Int64, #Int64._value +// CHECK: [[HOISTVAL:%.*]] = struct_extract [[ELT0]] : $Int64, #Int64._value // CHECK: br bb1([[PRELOAD]] : $Int64) // ...Loop // CHECK: bb1([[PHI:%.*]] : $Int64): @@ -1246,7 +1163,6 @@ bb3: // CHECK: [[ELT_1a:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, (Int64, Int64)), 1 // CHECK: [[ELT_1_0:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64), 0 // CHECK: [[V_1_0:%.*]] = load [[ELT_1_0]] : $*Int64 -// CHECK: [[ELT_1b:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, (Int64, Int64)), 1 // CHECK: [[ELT_1_1:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64), 1 // CHECK: [[V_1_1:%.*]] = load [[ELT_1_1]] : $*Int64 // CHECK: br bb1([[V_0:%.*]] : $Int64, [[V_1_0]] : $Int64) @@ -1374,7 +1290,7 @@ bb3: // CHECK: [[ELT_0:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 0 // CHECK: [[V0:%.*]] = load %6 : $*Int64 // CHECK: [[ARG0:%.*]] = tuple (%0 : $Int64, %0 : $Int64) -// CHECK: [[ARG0_0:%.*]] = tuple_extract %10 : $(Int64, Int64), 0 +// CHECK: [[ARG0_0:%.*]] = tuple_extract [[ARG0]] : $(Int64, Int64), 0 // CHECK: [[ARG1:%.*]] = tuple (%1 : $Int64, %1 : $Int64) // CHECK: br bb1([[V1]] : $(Int64, Int64)) // CHECK: bb1([[PHI:%.*]] : $(Int64, Int64)): From 5438deb3ffa17105446cbdd863d1a6ca107c74b1 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Thu, 31 Jul 2025 16:49:58 +0100 Subject: [PATCH 19/44] Add simple hoist. --- .../LoopInvariantCodeMotion.swift | 65 ++++++++++++------- test/SILOptimizer/licm.sil | 16 +++-- 2 files changed, 51 insertions(+), 30 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 8abef6773158d..27ff6d7bf8d7d 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -80,10 +80,11 @@ private func getWorkList(topLevelLoop: Loop, context: Context) -> Stack { struct MovableInstructions { var loadAndStoreAddrs: [AccessPath] = [] + var simplyHoistableInsts: [Instruction] = [] var loadsAndStores: [Instruction] = [] var hoistUp: [Instruction] = [] var sinkDown: [Instruction] = [] - var specialHoist: [Instruction] = [] + var scopedInsts: [Instruction] = [] } struct AnalyzedInstructions { @@ -223,7 +224,7 @@ private func analyzeLoop( hasOtherMemReadingInsts: &hasOtherMemReadingInsts ) case let refElementAddrInst as RefElementAddrInst: - movableInstructions.specialHoist.append(refElementAddrInst) + movableInstructions.simplyHoistableInsts.append(refElementAddrInst) case let condFailInst as CondFailInst: movableInstructions.hoistUp.append(condFailInst) condFailInst.analyzeSideEffects( @@ -361,7 +362,7 @@ private func analyzeLoop( aliasAnalysis: context.aliasAnalysis, domTree: context.dominatorTree ) { - movableInstructions.specialHoist.append(beginAccessInst) + movableInstructions.scopedInsts.append(beginAccessInst) } } @@ -373,14 +374,18 @@ private func optimizeLoop( movableInstructions: inout MovableInstructions, context: FunctionPassContext ) -> Bool { - if movableInstructions.hoistAllLoadsAndStores( + var changed = false + + // TODO: If we move tuple_element_addr and struct_element_addr instructions here, hoistAndSinkLoadsAndStores could converge after just one execution! + changed = movableInstructions.simpleHoistInstructions( + loop: loop, + context: context + ) || changed + + changed = movableInstructions.hoistAndSinkLoadsAndStores( outOf: loop, context: context - ) { - return true - } - - var changed = false + ) || changed changed = movableInstructions.hoistInstructions( loop: loop, @@ -444,13 +449,27 @@ extension MovableInstructions { return true } - mutating func hoistAllLoadsAndStores( + mutating func simpleHoistInstructions( + loop: Loop, + context: FunctionPassContext + ) -> Bool { + var changed = false + for inst in simplyHoistableInsts { + changed = inst.hoist( + outOf: loop, + context: context + ) || changed + } + return changed + } + + mutating func hoistAndSinkLoadsAndStores( outOf loop: Loop, context: FunctionPassContext ) -> Bool { var changed = false for accessPath in loadAndStoreAddrs { - changed = hoistAndSinkLoadsAndStores( + changed = hoistAndSinkLoadAndStore( outOf: loop, accessPath: accessPath, context: context @@ -500,25 +519,23 @@ extension MovableInstructions { loop: Loop, context: FunctionPassContext ) -> Bool { + guard !loop.hasNoExitBlocks else { + return false + } + var changed = false - for specialInst in specialHoist { - if specialInst is BeginAccessInst && loop.hasNoExitBlocks { - // TODO: We should move this as a precondition out of the loop once we remove RefElementAddrInst from here. + for specialInst in scopedInsts { + guard let beginAccessInst = specialInst as? BeginAccessInst else { continue } - guard - specialInst.hoist(outOf: loop, context: context) - else { + guard specialInst.hoist(outOf: loop, context: context) else { continue } - // TODO: This should probably be moved to the hoistUp collection. We should keep special hoist to only BeginAccessInst. - if let beginAccessInst = specialInst as? BeginAccessInst { - for endAccess in beginAccessInst.endAccessInstructions { - endAccess.sink(outOf: loop, context: context) - } + for endAccess in beginAccessInst.endAccessInstructions { + endAccess.sink(outOf: loop, context: context) } changed = true @@ -527,7 +544,7 @@ extension MovableInstructions { return changed } - private mutating func hoistAndSinkLoadsAndStores( + private mutating func hoistAndSinkLoadAndStore( outOf loop: Loop, accessPath: AccessPath, context: FunctionPassContext @@ -658,7 +675,7 @@ extension MovableInstructions { } } - if initialLoad.isTriviallyDead { + if initialLoad.isTriviallyDead || initialLoad.uses.isEmpty { context.erase(instruction: initialLoad) } diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index 392dc98850c99..0e969f6599505 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -1048,6 +1048,7 @@ struct State { // CHECK: [[PRELOAD:%.*]] = load [[PRELOADADR]] : $*Int64 // ...Split element 0 // CHECK: [[ELT0:%.*]] = load [[HOISTADR]] : $*Int64 +// CHECK: [[HOISTVAL:%.*]] = struct_extract [[ELT0]] : $Int64, #Int64._value // ...Split element 2 // CHECK: [[SPLIT2:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 2 // CHECK: [[ELT2:%.*]] = load [[SPLIT2]] : $*Int64 @@ -1055,7 +1056,6 @@ struct State { // CHECK: [[SINGLEADR:%.*]] = struct_element_addr %{{.*}} : $*State, #State.singleValue // CHECK: [[SINGLEVAL:%.*]] = load [[SINGLEADR]] : $*Int64 // ...Hoisted element 0 -// CHECK: [[HOISTVAL:%.*]] = struct_extract [[ELT0]] : $Int64, #Int64._value // CHECK: br bb1([[PRELOAD]] : $Int64) // ...Loop // CHECK: bb1([[PHI:%.*]] : $Int64): @@ -1287,11 +1287,11 @@ bb3: // CHECK: bb0(%0 : $Int64, %1 : $Int64, %2 : $Builtin.RawPointer): // CHECK: [[ELT_1:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 1 // CHECK: [[V1:%.*]] = load %4 : $*(Int64, Int64) -// CHECK: [[ELT_0:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 0 -// CHECK: [[V0:%.*]] = load %6 : $*Int64 // CHECK: [[ARG0:%.*]] = tuple (%0 : $Int64, %0 : $Int64) -// CHECK: [[ARG0_0:%.*]] = tuple_extract [[ARG0]] : $(Int64, Int64), 0 // CHECK: [[ARG1:%.*]] = tuple (%1 : $Int64, %1 : $Int64) +// CHECK: [[ELT_0:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 0 +// CHECK: [[V0:%.*]] = load %9 : $*Int64 +// CHECK: [[ARG0_0:%.*]] = tuple_extract [[ARG0]] : $(Int64, Int64), 0 // CHECK: br bb1([[V1]] : $(Int64, Int64)) // CHECK: bb1([[PHI:%.*]] : $(Int64, Int64)): // CHECK: [[LOOPVAL:%.*]] = tuple ([[V0]] : $Int64, [[PHI]] : $(Int64, Int64)) @@ -1339,8 +1339,10 @@ class C {} // This won't be hoisted because we can't find a base to check if it is invariant // CHECK-LABEL: sil @testLoopInvariantStoreNoBase1 : +// CHECK: bb3(%11 : $Builtin.RawPointer): +// CHECK-NOT: load // CHECK: bb6: -// CHECK-NOT: store +// CHECK: store // CHECK-LABEL: } // end sil function 'testLoopInvariantStoreNoBase1' sil @testLoopInvariantStoreNoBase1 : $@convention(thin) (Builtin.BridgeObject, Double) -> () { bb0(%0 : $Builtin.BridgeObject, %1 : $Double): @@ -1376,8 +1378,10 @@ bb6: // This won't be hoisted because we can't find a base to check if it is invariant // CHECK-LABEL: sil @testLoopInvariantStoreNoBase2 : +// CHECK: bb3(%11 : $Builtin.RawPointer): +// CHECK-NOT: load // CHECK: bb6: -// CHECK-NOT: store +// CHECK: store // CHECK-LABEL: } // end sil function 'testLoopInvariantStoreNoBase2' sil @testLoopInvariantStoreNoBase2 : $@convention(thin) (Builtin.BridgeObject, Double) -> () { bb0(%0 : $Builtin.BridgeObject, %1 : $Double): From bb5fdcb7d69536807f503745593f2ab4959bde80 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Thu, 31 Jul 2025 18:16:45 +0100 Subject: [PATCH 20/44] Fix end access hoisting by checking whether instructions are inside the scope. --- .../LoopInvariantCodeMotion.swift | 30 +++++++++--- test/SILOptimizer/licm.sil | 46 ------------------- test/SILOptimizer/licm_apply.sil | 2 +- test/SILOptimizer/licm_exclusivity.sil | 2 +- test/SILOptimizer/licm_multiend.sil | 2 +- 5 files changed, 27 insertions(+), 55 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 27ff6d7bf8d7d..b4a34ac994a0d 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -376,7 +376,7 @@ private func optimizeLoop( ) -> Bool { var changed = false - // TODO: If we move tuple_element_addr and struct_element_addr instructions here, hoistAndSinkLoadsAndStores could converge after just one execution! + // TODO: If we hoist tuple_element_addr and struct_element_addr instructions here, hoistAndSinkLoadsAndStores could converge after just one execution! changed = movableInstructions.simpleHoistInstructions( loop: loop, context: context @@ -732,6 +732,27 @@ extension Loop { } extension Instruction { + fileprivate func dominates( + _ otherInst: Instruction, + _ domTree: DominatorTree + ) -> Bool { + if parentBlock == otherInst.parentBlock { + for inst in parentBlock.instructions { + if inst == self { + return true + } else if inst == otherInst { + break + } + } + return false + } else { + return parentBlock.dominates( + otherInst.parentBlock, + domTree + ) + } + } + fileprivate func analyzeSideEffects( analyzedInstructions: inout AnalyzedInstructions, blockSideEffects: inout Stack, @@ -1051,12 +1072,9 @@ extension BeginAccessInst { otherInst: Instruction, domTree: DominatorTree ) -> Bool { - return self.parentBlock.strictlyDominates( - otherInst.parentBlock, - domTree - ) && self.endAccessInstructions + return self.dominates(otherInst, domTree) && endAccessInstructions .allSatisfy { endAccessInst in - otherInst.parentBlock.dominates(endAccessInst.parentBlock, domTree) + otherInst.dominates(endAccessInst, domTree) } } diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index 0e969f6599505..8345589642cf3 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -1237,52 +1237,6 @@ bb3: return %val1 : $(Int64, (Int64, Int64)) } -// Two wide loads. The first can be successfully split and the second -// half hoisted. The second cannot be split because of a pointer -// cast. Make sure two remaining loads and the store are still in the loop. -// -// CHECK-LABEL: sil hidden @testSplitNonStandardProjection : $@convention(method) (Int64, Builtin.RawPointer) -> ((Int64, (Int64, Int64)), (Int64, Int64)) { -// CHECK: bb0(%0 : $Int64, %1 : $Builtin.RawPointer): -// -// The first load was split, so one half is hoisted. -// CHECK: [[V1:%.*]] = load %{{.*}} : $*Int64 -// CHECK: br bb1 -// CHECK: bb1: -// CHECK: [[V0:%.*]] = load %{{.*}} : $*Int64 -// CHECK: [[INNER:%.*]] = tuple ([[V0]] : $Int64, [[V1]] : $Int64) -// CHECK: store %0 to %{{.*}} : $*Int64 -// CHECK: [[OUTER:%.*]] = load %{{.*}} : $*(Int64, (Int64, Int64)) -// CHECK: cond_br undef, bb2, bb3 -// CHECK: bb2: -// CHECK: br bb1 -// CHECK: bb3: -// CHECK: [[RESULT:%.*]] = tuple ([[OUTER]] : $(Int64, (Int64, Int64)), [[INNER]] : $(Int64, Int64)) -// CHECK: return [[RESULT]] : $((Int64, (Int64, Int64)), (Int64, Int64)) -// CHECK-LABEL: } // end sil function 'testSplitNonStandardProjection' -sil hidden @testSplitNonStandardProjection : $@convention(method) (Int64, Builtin.RawPointer) -> ((Int64, (Int64, Int64)), (Int64, Int64)) { -bb0(%0 : $Int64, %1 : $Builtin.RawPointer): - %outerAddr1 = pointer_to_address %1 : $Builtin.RawPointer to $*(Int64, (Int64, Int64)) - br bb1 - -bb1: - %elt1 = tuple_element_addr %outerAddr1 : $*(Int64, (Int64, Int64)), 1 - %ptr = address_to_pointer %elt1 : $*(Int64, Int64) to $Builtin.RawPointer - %ptrAdr = pointer_to_address %ptr : $Builtin.RawPointer to [strict] $*(Int64, Int64) - %val2 = load %ptrAdr : $*(Int64, Int64) - %eltptr0 = tuple_element_addr %ptrAdr : $*(Int64, Int64), 0 - store %0 to %eltptr0 : $*Int64 - // Process the outermost load after splitting the inner load - %val1 = load %outerAddr1 : $*(Int64, (Int64, Int64)) - cond_br undef, bb2, bb3 - -bb2: - br bb1 - -bb3: - %result = tuple (%val1 : $(Int64, (Int64, Int64)), %val2 : $(Int64, Int64)) - return %result : $((Int64, (Int64, Int64)), (Int64, Int64)) -} - // CHECK-LABEL: sil shared @testSameTwoStores : $@convention(method) (Int64, Int64, Builtin.RawPointer) -> (Int64, (Int64, Int64)) { // CHECK: bb0(%0 : $Int64, %1 : $Int64, %2 : $Builtin.RawPointer): // CHECK: [[ELT_1:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 1 diff --git a/test/SILOptimizer/licm_apply.sil b/test/SILOptimizer/licm_apply.sil index df5a5381a2731..77fe0bf6b5a4f 100644 --- a/test/SILOptimizer/licm_apply.sil +++ b/test/SILOptimizer/licm_apply.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -compute-side-effects -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -compute-side-effects -loop-invariant-code-motion | %FileCheck %s // REQUIRES: swift_in_compiler diff --git a/test/SILOptimizer/licm_exclusivity.sil b/test/SILOptimizer/licm_exclusivity.sil index bd7049cc7bc06..fcfe9770af4be 100644 --- a/test/SILOptimizer/licm_exclusivity.sil +++ b/test/SILOptimizer/licm_exclusivity.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=checked -enable-sil-verify-all %s -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=checked -enable-sil-verify-all %s -loop-invariant-code-motion | %FileCheck %s sil_stage canonical diff --git a/test/SILOptimizer/licm_multiend.sil b/test/SILOptimizer/licm_multiend.sil index 32d83b3d14ad6..5a22419f66b5f 100644 --- a/test/SILOptimizer/licm_multiend.sil +++ b/test/SILOptimizer/licm_multiend.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -loop-invariant-code-motion | %FileCheck %s // REQUIRES: PTRSIZE=64 // REQUIRES: OS=macosx From 11f1e7706b968c06dd321439b77791f4fa3f1bd6 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Fri, 1 Aug 2025 15:11:43 +0100 Subject: [PATCH 21/44] Fix side effect handling. Reintroduce non-standard projection test case. --- .../LoopInvariantCodeMotion.swift | 6 +-- test/SILOptimizer/licm.sil | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index b4a34ac994a0d..708b46332f06f 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -1031,19 +1031,19 @@ extension ApplyInst { aliasAnalysis: AliasAnalysis, calleeAnalysis: CalleeAnalysis ) -> Bool { - guard calleeAnalysis.getSideEffects(ofApply: self).memory == .noEffects else { + guard calleeAnalysis.getSideEffects(ofApply: self).memory != .noEffects else { return false } for sideEffect in sideEffects { switch sideEffect { case let storeInst as StoreInst: - if storeInst.storeOwnership == .assign && + if storeInst.storeOwnership == .assign || mayRead(fromAddress: storeInst.destination, aliasAnalysis) { return true } case let copyAddrInst as CopyAddrInst: - if !copyAddrInst.isInitializationOfDestination && + if !copyAddrInst.isInitializationOfDestination || mayRead(fromAddress: copyAddrInst.destination, aliasAnalysis) { return true } diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index 8345589642cf3..404aea3119a38 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -1237,6 +1237,51 @@ bb3: return %val1 : $(Int64, (Int64, Int64)) } +// Split two wide loads. +// +// CHECK-LABEL: sil hidden @testSplitNonStandardProjection : $@convention(method) (Int64, Builtin.RawPointer) -> ((Int64, (Int64, Int64)), (Int64, Int64)) { +// CHECK: bb0(%0 : $Int64, %1 : $Builtin.RawPointer): +// +// CHECK: [[ADDR1:%.*]] = tuple_element_addr %{{.*}}, 0 +// CHECK: [[V1:%.*]] = load [[ADDR1]] : $*Int64 +// CHECK: [[ADDR0:%.*]] = tuple_element_addr %{{.*}}, 1 +// CHECK: [[V0:%.*]] = load [[ADDR0]] : $*Int64 +// CHECK: [[OUTER:%.*]] = tuple (%{{.*}} : $Int64, %{{.*}} : $(Int64, Int64)) +// CHECK: br bb1([[V1]] : $Int64) +// CHECK: bb1([[PHI:%.*]] : $Int64): +// CHECK: [[INNER:%.*]] = tuple ([[PHI]] : $Int64, [[V0]] : $Int64) +// CHECK: cond_br undef, bb2, bb3 +// CHECK: bb2: +// CHECK: br bb1(%0 : $Int64) +// CHECK: bb3: +// CHECK: store %0 to [[ADDR1]] : $*Int64 +// CHECK: [[RESULT:%.*]] = tuple ([[OUTER]] : $(Int64, (Int64, Int64)), [[INNER]] : $(Int64, Int64)) +// CHECK: return [[RESULT]] : $((Int64, (Int64, Int64)), (Int64, Int64)) +// CHECK-LABEL: } // end sil function 'testSplitNonStandardProjection' +sil hidden @testSplitNonStandardProjection : $@convention(method) (Int64, Builtin.RawPointer) -> ((Int64, (Int64, Int64)), (Int64, Int64)) { +bb0(%0 : $Int64, %1 : $Builtin.RawPointer): + %outerAddr1 = pointer_to_address %1 : $Builtin.RawPointer to $*(Int64, (Int64, Int64)) + br bb1 + +bb1: + %elt1 = tuple_element_addr %outerAddr1 : $*(Int64, (Int64, Int64)), 1 + %ptr = address_to_pointer %elt1 : $*(Int64, Int64) to $Builtin.RawPointer + %ptrAdr = pointer_to_address %ptr : $Builtin.RawPointer to [strict] $*(Int64, Int64) + %val2 = load %ptrAdr : $*(Int64, Int64) + %eltptr0 = tuple_element_addr %ptrAdr : $*(Int64, Int64), 0 + store %0 to %eltptr0 : $*Int64 + // Process the outermost load after splitting the inner load + %val1 = load %outerAddr1 : $*(Int64, (Int64, Int64)) + cond_br undef, bb2, bb3 + +bb2: + br bb1 + +bb3: + %result = tuple (%val1 : $(Int64, (Int64, Int64)), %val2 : $(Int64, Int64)) + return %result : $((Int64, (Int64, Int64)), (Int64, Int64)) +} + // CHECK-LABEL: sil shared @testSameTwoStores : $@convention(method) (Int64, Int64, Builtin.RawPointer) -> (Int64, (Int64, Int64)) { // CHECK: bb0(%0 : $Int64, %1 : $Int64, %2 : $Builtin.RawPointer): // CHECK: [[ELT_1:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 1 From 1a7bf6dd64b6ece231652e23887e223755dc5c46 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Fri, 1 Aug 2025 16:45:17 +0100 Subject: [PATCH 22/44] Fix merge artifacts. --- .../LoopInvariantCodeMotion.swift | 14 +++ .../PassManager/FunctionPassContext.swift | 19 ++-- .../Sources/SIL/BasicBlock.swift | 17 ---- .../Sources/SIL/Instruction.swift | 5 + include/swift/SIL/SILBridging.h | 39 ++++++++ include/swift/SIL/SILBridgingImpl.h | 4 + .../swift/SILOptimizer/OptimizerBridging.h | 95 ------------------- .../SILOptimizer/OptimizerBridgingImpl.h | 4 - 8 files changed, 72 insertions(+), 125 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 708b46332f06f..4905fed93041b 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import SIL +import OptimizerBridging let loopInvariantCodeMotionPass = FunctionPass( name: "loop-invariant-code-motion" @@ -929,6 +930,19 @@ extension Instruction { ) } } + + func getArraySemanticsCallKind() -> BridgedArrayCallKind { + return BridgedPassContext.getArraySemanticsCallKind(self.bridged) + } + + func canHoistArraySemanticsCall(to toInst: Instruction, _ context: FunctionPassContext) -> Bool { + return context.bridgedPassContext.canHoistArraySemanticsCall(self.bridged, toInst.bridged) + } + + func hoistArraySemanticsCall(before toInst: Instruction, _ context: some MutatingContext) { + context.bridgedPassContext.hoistArraySemanticsCall(self.bridged, toInst.bridged) // Internally updates dom tree. + context.notifyInstructionsChanged() + } } extension UnaryInstruction { diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift index 18620eea7e914..d74055f4a2051 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift @@ -45,6 +45,11 @@ struct FunctionPassContext : MutatingContext { let bridgedPDT = bridgedPassContext.getPostDomTree() return PostDominatorTree(bridged: bridgedPDT) } + + var loopTree: LoopTree { + let bridgedLT = bridgedPassContext.getLoopTree() + return LoopTree(bridged: bridgedLT, context: self) + } func loadFunction(name: StaticString, loadCalleesRecursively: Bool) -> Function? { return name.withUTF8Buffer { (nameBuffer: UnsafeBufferPointer) in @@ -92,15 +97,11 @@ struct FunctionPassContext : MutatingContext { return String(taking: bridgedPassContext.mangleOutlinedVariable(function.bridged)) } - func mangle(withClosureArguments closureArgs: [Value], closureArgIndices: [Int], from applySiteCallee: Function) -> String { - closureArgs.withBridgedValues { bridgedClosureArgsRef in - closureArgIndices.withBridgedArrayRef{bridgedClosureArgIndicesRef in - String(taking: bridgedPassContext.mangleWithClosureArgs( - bridgedClosureArgsRef, - bridgedClosureArgIndicesRef, - applySiteCallee.bridged - )) - } + func mangle(withClosureArguments closureArgs: [(argumentIndex: Int, argumentValue: Value)], + from applySiteCallee: Function + ) -> String { + closureArgs.withBridgedArrayRef{ bridgedClosureArgs in + String(taking: bridgedPassContext.mangleWithClosureArgs(bridgedClosureArgs, applySiteCallee.bridged)) } } diff --git a/SwiftCompilerSources/Sources/SIL/BasicBlock.swift b/SwiftCompilerSources/Sources/SIL/BasicBlock.swift index d067ce6771084..c543559c24489 100644 --- a/SwiftCompilerSources/Sources/SIL/BasicBlock.swift +++ b/SwiftCompilerSources/Sources/SIL/BasicBlock.swift @@ -153,15 +153,6 @@ final public class BasicBlock : CustomStringConvertible, HasShortDescription, Ha return false } } - - /// The index of the basic block in its function. - /// This has O(n) complexity. Only use it for debugging - public var index: Int { - for (idx, block) in parentFunction.blocks.enumerated() { - if block == self { return idx } - } - fatalError() - } public func isCriticalEdge(edgeIndex: Int) -> Bool { if terminator.successors.count <= 1 { @@ -170,14 +161,6 @@ final public class BasicBlock : CustomStringConvertible, HasShortDescription, Ha return !terminator.successors[edgeIndex].hasSinglePredecessor } } - - public var name: String { "bb\(index)" } - - public static func == (lhs: BasicBlock, rhs: BasicBlock) -> Bool { lhs === rhs } - - public func hash(into hasher: inout Hasher) { - hasher.combine(ObjectIdentifier(self)) - } } /// The list of instructions in a BasicBlock. diff --git a/SwiftCompilerSources/Sources/SIL/Instruction.swift b/SwiftCompilerSources/Sources/SIL/Instruction.swift index d37d88f08380e..073c4274ef785 100644 --- a/SwiftCompilerSources/Sources/SIL/Instruction.swift +++ b/SwiftCompilerSources/Sources/SIL/Instruction.swift @@ -96,6 +96,11 @@ public class Instruction : CustomStringConvertible, Hashable { BridgedContext.moveInstructionBefore(bridged, otherInstruction.bridged) context.notifyInstructionsChanged() } + + public final func copy(before otherInstruction: Instruction, _ context: some MutatingContext) { + BridgedContext.copyInstructionBefore(bridged, otherInstruction.bridged) + context.notifyInstructionsChanged() + } public var mayTrap: Bool { false } diff --git a/include/swift/SIL/SILBridging.h b/include/swift/SIL/SILBridging.h index 413618f6fbeb4..66913e6a79598 100644 --- a/include/swift/SIL/SILBridging.h +++ b/include/swift/SIL/SILBridging.h @@ -60,6 +60,8 @@ class SymbolicValueBumpAllocator; class ConstExprEvaluator; class SILWitnessTable; class SILDefaultWitnessTable; +class SILLoopInfo; +class SILLoop; class SILDebugLocation; class NominalTypeDecl; class VarDecl; @@ -80,6 +82,42 @@ class FixedSizeSlabPayload; class FixedSizeSlab; } +struct BridgedLoop { + swift::SILLoop * _Nonnull l; + + BRIDGED_INLINE SwiftInt getInnerLoopCount() const; + BRIDGED_INLINE BridgedLoop getInnerLoop(SwiftInt index) const; + + BRIDGED_INLINE SwiftInt getBasicBlockCount() const; + BRIDGED_INLINE BridgedBasicBlock getBasicBlock(SwiftInt index) const; + + BRIDGED_INLINE OptionalBridgedBasicBlock getPreheader() const; + BRIDGED_INLINE BridgedBasicBlock getHeader() const; +}; + +enum class BridgedArrayCallKind { + kNone = 0, + kArrayPropsIsNativeTypeChecked, + kCheckSubscript, + kCheckIndex, + kGetCount, + kGetCapacity, + kGetElement, + kGetElementAddress, + kMakeMutable, + kEndMutation, + kMutateUnknown, + kReserveCapacityForAppend, + kWithUnsafeMutableBufferPointer, + kAppendContentsOf, + kAppendElement, + kArrayInit, + kArrayInitEmpty, + kArrayUninitialized, + kArrayUninitializedIntrinsic, + kArrayFinalizeIntrinsic +}; + bool swiftModulesInitialized(); void registerBridgedClass(BridgedStringRef className, SwiftMetatype metatype); @@ -1459,6 +1497,7 @@ struct BridgedContext { BRIDGED_INLINE void eraseInstruction(BridgedInstruction inst, bool salvageDebugInfo) const; BRIDGED_INLINE void eraseBlock(BridgedBasicBlock block) const; static BRIDGED_INLINE void moveInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst); + static BRIDGED_INLINE void copyInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst); // SSAUpdater diff --git a/include/swift/SIL/SILBridgingImpl.h b/include/swift/SIL/SILBridgingImpl.h index 128e29a384172..f007b7def0d46 100644 --- a/include/swift/SIL/SILBridgingImpl.h +++ b/include/swift/SIL/SILBridgingImpl.h @@ -2921,6 +2921,10 @@ void BridgedContext::moveInstructionBefore(BridgedInstruction inst, BridgedInstr swift::SILBasicBlock::moveInstruction(inst.unbridged(), beforeInst.unbridged()); } +void BridgedContext::copyInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst) { + inst.unbridged()->clone(beforeInst.unbridged()); +} + OptionalBridgedFunction BridgedContext::lookupStdlibFunction(BridgedStringRef name) const { return {context->lookupStdlibFunction(name.unbridged())}; } diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index bc8903833875b..ec69a0149ca25 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -128,19 +128,6 @@ struct BridgedPostDomTree { BRIDGED_INLINE bool postDominates(BridgedBasicBlock dominating, BridgedBasicBlock dominated) const; }; -struct BridgedLoop { - swift::SILLoop * _Nonnull l; - - BRIDGED_INLINE SwiftInt getInnerLoopCount() const; - BRIDGED_INLINE BridgedLoop getInnerLoop(SwiftInt index) const; - - BRIDGED_INLINE SwiftInt getBasicBlockCount() const; - BRIDGED_INLINE BridgedBasicBlock getBasicBlock(SwiftInt index) const; - - BRIDGED_INLINE OptionalBridgedBasicBlock getPreheader() const; - BRIDGED_INLINE BridgedBasicBlock getHeader() const; -}; - struct BridgedLoopTree { swift::SILLoopInfo * _Nonnull li; @@ -150,88 +137,6 @@ struct BridgedLoopTree { SWIFT_IMPORT_UNSAFE BRIDGED_INLINE OptionalBridgedBasicBlock splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const; }; -enum class BridgedArrayCallKind { - kNone = 0, - kArrayPropsIsNativeTypeChecked, - kCheckSubscript, - kCheckIndex, - kGetCount, - kGetCapacity, - kGetElement, - kGetElementAddress, - kMakeMutable, - kEndMutation, - kMutateUnknown, - kReserveCapacityForAppend, - kWithUnsafeMutableBufferPointer, - kAppendContentsOf, - kAppendElement, - kArrayInit, - kArrayInitEmpty, - kArrayUninitialized, - kArrayUninitializedIntrinsic, - kArrayFinalizeIntrinsic -}; - -struct BridgedUtilities { - typedef void (* _Nonnull VerifyFunctionFn)(BridgedPassContext, BridgedFunction); - typedef void (* _Nonnull UpdateFunctionFn)(BridgedPassContext, BridgedFunction); - typedef void (* _Nonnull UpdatePhisFn)(BridgedPassContext, BridgedArrayRef); - - static void registerVerifier(VerifyFunctionFn verifyFunctionFn); - static void registerPhiUpdater(UpdateFunctionFn updateBorrowedFromFn, - UpdatePhisFn updateBorrowedFromPhisFn, - UpdatePhisFn replacePhisWithIncomingValuesFn); -}; - -struct BridgedBasicBlockSet { - swift::BasicBlockSet * _Nonnull set; - - BRIDGED_INLINE bool contains(BridgedBasicBlock block) const; - BRIDGED_INLINE bool insert(BridgedBasicBlock block) const; - BRIDGED_INLINE void erase(BridgedBasicBlock block) const; - SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedFunction getFunction() const; -}; - -struct BridgedNodeSet { - swift::NodeSet * _Nonnull set; - - BRIDGED_INLINE bool containsValue(BridgedValue value) const; - BRIDGED_INLINE bool insertValue(BridgedValue value) const; - BRIDGED_INLINE void eraseValue(BridgedValue value) const; - BRIDGED_INLINE bool containsInstruction(BridgedInstruction inst) const; - BRIDGED_INLINE bool insertInstruction(BridgedInstruction inst) const; - BRIDGED_INLINE void eraseInstruction(BridgedInstruction inst) const; - SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedFunction getFunction() const; -}; - -struct BridgedOperandSet { - swift::OperandSet * _Nonnull set; - - BRIDGED_INLINE bool contains(BridgedOperand operand) const; - BRIDGED_INLINE bool insert(BridgedOperand operand) const; - BRIDGED_INLINE void erase(BridgedOperand operand) const; - SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedFunction getFunction() const; -}; - -struct BridgedCloner { - swift::BridgedClonerImpl * _Nonnull cloner; - - BridgedCloner(BridgedGlobalVar var, BridgedPassContext context); - BridgedCloner(BridgedInstruction inst, BridgedPassContext context); - BridgedCloner(BridgedFunction emptyFunction, BridgedPassContext context); - void destroy(BridgedPassContext context); - SWIFT_IMPORT_UNSAFE BridgedFunction getCloned() const; - SWIFT_IMPORT_UNSAFE BridgedBasicBlock getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const; - void cloneFunctionBody(BridgedFunction originalFunction, BridgedBasicBlock clonedEntryBlock, - BridgedValueArray clonedEntryBlockArgs) const; - void cloneFunctionBody(BridgedFunction originalFunction) const; - SWIFT_IMPORT_UNSAFE BridgedValue getClonedValue(BridgedValue v); - bool isValueCloned(BridgedValue v) const; - void recordClonedInstruction(BridgedInstruction origInst, BridgedInstruction clonedInst) const; - BridgedInstruction clone(BridgedInstruction inst); -}; - struct BridgedPassContext { swift::SwiftPassInvocation * _Nonnull invocation; diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index 6798dae1239b5..f056ccd4f8142 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -335,10 +335,6 @@ void BridgedPassContext::copyInstructionBefore(BridgedInstruction inst, BridgedI inst.unbridged()->clone(beforeInst.unbridged()); } -BridgedValue BridgedPassContext::getSILUndef(BridgedType type) const { - return {swift::SILUndef::get(invocation->getFunction(), type.unbridged())}; -} - bool BridgedPassContext::eliminateDeadAllocations(BridgedFunction f) const { return swift::eliminateDeadAllocations(f.getFunction(), this->getDomTree().di); From cd4bcdffa48f533adeace28f5b404effaf053ff4 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Mon, 4 Aug 2025 10:47:38 +0100 Subject: [PATCH 23/44] Fix build artifacts related to the new Cloner. --- .../ClosureSpecialization.swift | 2 +- .../Optimizer/Utilities/CMakeLists.txt | 5 -- .../Sources/Optimizer/Utilities/Cloner.swift | 25 +++--- include/swift/SIL/SILBridging.h | 13 --- .../swift/SILOptimizer/OptimizerBridging.h | 18 +++++ .../SILOptimizer/OptimizerBridgingImpl.h | 72 ----------------- lib/SIL/Utils/SILBridging.cpp | 79 ------------------- 7 files changed, 32 insertions(+), 182 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift index abe786a4503c8..4880f815ae20b 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift @@ -838,7 +838,7 @@ private extension Cloner where Context == FunctionPassContext { } if (self.context.needFixStackNesting) { - self.targetFunction.fixStackNesting(self.context) + self.context.fixStackNesting(in: targetFunction) } } } diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt index dc93336f9768d..55d6d3c6fbc1d 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt @@ -8,7 +8,6 @@ swift_compiler_sources(Optimizer AddressUtils.swift - BorrowUtils.swift Cloner.swift Devirtualization.swift EscapeUtils.swift @@ -19,8 +18,4 @@ swift_compiler_sources(Optimizer LocalVariableUtils.swift OptUtils.swift OwnershipLiveness.swift - PhiUpdater.swift - SSAUpdater.swift - Test.swift - Verifier.swift ) diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift index 802fbd637a4e0..d2bd929ecba1c 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift @@ -12,6 +12,7 @@ import OptimizerBridging import SIL +import SILBridging /// Clones the initializer value of a GlobalVariable. /// @@ -29,25 +30,25 @@ struct Cloner { let target: Target init(cloneToGlobal: GlobalVariable, _ context: Context) { - self.bridged = BridgedCloner(cloneToGlobal.bridged, context._bridged) + self.bridged = BridgedCloner(cloneToGlobal.bridged, context.bridgedPassContext) self.context = context self.target = .global(cloneToGlobal) } init(cloneBefore inst: Instruction, _ context: Context) { - self.bridged = BridgedCloner(inst.bridged, context._bridged) + self.bridged = BridgedCloner(inst.bridged, context.bridgedPassContext) self.context = context self.target = .function(inst.parentFunction) } init(cloneToEmptyFunction: Function, _ context: Context) where Context == FunctionPassContext { - self.bridged = BridgedCloner(cloneToEmptyFunction.bridged, context._bridged) + self.bridged = BridgedCloner(cloneToEmptyFunction.bridged, context.bridgedPassContext) self.context = context self.target = .function(cloneToEmptyFunction) } mutating func deinitialize() { - bridged.destroy(context._bridged) + bridged.destroy(context.bridgedPassContext) } var targetFunction: Function { @@ -88,23 +89,23 @@ struct Cloner { } fatalError("unexpected instruction kind") } - + mutating func cloneUseDefChain(addr: Value, checkBase: (Value) -> Bool) -> Value? { // MARK: Hacky temp fix if addr is AllocStackInst { return nil } - + guard !checkBase(addr) else { guard let inst = addr as? Instruction else { // TODO: Might have to additionally register like the instruction below. return addr } - + bridged.recordClonedInstruction(inst.bridged, inst.bridged) return addr } - + switch addr { // The cloner does not currently know how to create compensating // end_borrows or fix mark_dependence operands. @@ -112,19 +113,19 @@ struct Cloner { case let singleValueInstruction as SingleValueInstruction: // TODO: Double check whether correct guard let sourceOperand = singleValueInstruction.operands.first else { return nil } - + return cloneProjection(projectAddr: singleValueInstruction, sourceOperand: sourceOperand, checkBase: checkBase) default: return nil } } - + private func shouldClone(_ value: Value) -> Bool { switch value { case is StructElementAddrInst, is TupleElementAddrInst, is IndexAddrInst, is TailAddrInst, is InitEnumDataAddrInst, is OpenExistentialAddrInst, is UncheckedTakeEnumDataAddrInst, is ProjectBoxInst, is ProjectBlockStorageInst, is MoveOnlyWrapperToCopyableAddrInst, is CopyableToMoveOnlyWrapperAddrInst, is MoveOnlyWrapperToCopyableBoxInst, is UncheckedAddrCastInst, is AddressToPointerInst, is PointerToAddressInst, is MarkUninitializedInst, is MarkUnresolvedReferenceBindingInst, is DropDeinitInst, is MarkUnresolvedReferenceBindingInst, is MarkDependenceInst, is CopyValueInst, is BeginBorrowInst, is StoreBorrowInst: return true default: return false } } - + private mutating func cloneProjection( projectAddr: SingleValueInstruction, sourceOperand: Operand, @@ -136,7 +137,7 @@ struct Cloner { ) else { return nil } - + // for op in projectAddr.operands { // _ = cloneUseDefChain(addr: op.value, checkBase: checkBase) // } diff --git a/include/swift/SIL/SILBridging.h b/include/swift/SIL/SILBridging.h index 66913e6a79598..38d264a465831 100644 --- a/include/swift/SIL/SILBridging.h +++ b/include/swift/SIL/SILBridging.h @@ -77,7 +77,6 @@ class SILLocation; class BasicBlockSet; class NodeSet; class OperandSet; -class ClonerWithFixedLocation; class FixedSizeSlabPayload; class FixedSizeSlab; } @@ -1409,18 +1408,6 @@ struct BridgedOperandSet { SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedFunction getFunction() const; }; -struct BridgedCloner { - swift::ClonerWithFixedLocation * _Nonnull cloner; - - BridgedCloner(BridgedGlobalVar var, BridgedContext context); - BridgedCloner(BridgedInstruction inst, BridgedContext context); - void destroy(BridgedContext context); - SWIFT_IMPORT_UNSAFE BridgedValue getClonedValue(BridgedValue v); - bool isValueCloned(BridgedValue v) const; - void clone(BridgedInstruction inst); - void recordFoldedValue(BridgedValue origValue, BridgedValue mappedValue); -}; - struct BridgedContext { swift::SILContext * _Nonnull context; diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index ec69a0149ca25..a13b71f52dcd8 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -137,6 +137,24 @@ struct BridgedLoopTree { SWIFT_IMPORT_UNSAFE BRIDGED_INLINE OptionalBridgedBasicBlock splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const; }; +struct BridgedCloner { + swift::BridgedClonerImpl * _Nonnull cloner; + + BridgedCloner(BridgedGlobalVar var, BridgedPassContext context); + BridgedCloner(BridgedInstruction inst, BridgedPassContext context); + BridgedCloner(BridgedFunction emptyFunction, BridgedPassContext context); + void destroy(BridgedPassContext context); + SWIFT_IMPORT_UNSAFE BridgedFunction getCloned() const; + SWIFT_IMPORT_UNSAFE BridgedBasicBlock getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const; + void cloneFunctionBody(BridgedFunction originalFunction, BridgedBasicBlock clonedEntryBlock, + BridgedValueArray clonedEntryBlockArgs) const; + void cloneFunctionBody(BridgedFunction originalFunction) const; + SWIFT_IMPORT_UNSAFE BridgedValue getClonedValue(BridgedValue v); + bool isValueCloned(BridgedValue v) const; + void recordClonedInstruction(BridgedInstruction origInst, BridgedInstruction clonedInst) const; + BridgedInstruction clone(BridgedInstruction inst); +}; + struct BridgedPassContext { swift::SwiftPassInvocation * _Nonnull invocation; diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index f056ccd4f8142..5b8773cc8944d 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -137,78 +137,6 @@ BridgedBasicBlock BridgedLoop::getHeader() const { return {l->getHeader()}; } -//===----------------------------------------------------------------------===// -// BridgedBasicBlockSet -//===----------------------------------------------------------------------===// - -bool BridgedBasicBlockSet::contains(BridgedBasicBlock block) const { - return set->contains(block.unbridged()); -} - -bool BridgedBasicBlockSet::insert(BridgedBasicBlock block) const { - return set->insert(block.unbridged()); -} - -void BridgedBasicBlockSet::erase(BridgedBasicBlock block) const { - set->erase(block.unbridged()); -} - -BridgedFunction BridgedBasicBlockSet::getFunction() const { - return {set->getFunction()}; -} - -//===----------------------------------------------------------------------===// -// BridgedNodeSet -//===----------------------------------------------------------------------===// - -bool BridgedNodeSet::containsValue(BridgedValue value) const { - return set->contains(value.getSILValue()); -} - -bool BridgedNodeSet::insertValue(BridgedValue value) const { - return set->insert(value.getSILValue()); -} - -void BridgedNodeSet::eraseValue(BridgedValue value) const { - set->erase(value.getSILValue()); -} - -bool BridgedNodeSet::containsInstruction(BridgedInstruction inst) const { - return set->contains(inst.unbridged()->asSILNode()); -} - -bool BridgedNodeSet::insertInstruction(BridgedInstruction inst) const { - return set->insert(inst.unbridged()->asSILNode()); -} - -void BridgedNodeSet::eraseInstruction(BridgedInstruction inst) const { - set->erase(inst.unbridged()->asSILNode()); -} - -BridgedFunction BridgedNodeSet::getFunction() const { - return {set->getFunction()}; -} - -//===----------------------------------------------------------------------===// -// BridgedOperandSet -//===----------------------------------------------------------------------===// - -bool BridgedOperandSet::contains(BridgedOperand operand) const { - return set->contains(operand.op); -} - -bool BridgedOperandSet::insert(BridgedOperand operand) const { - return set->insert(operand.op); -} - -void BridgedOperandSet::erase(BridgedOperand operand) const { - set->erase(operand.op); -} - -BridgedFunction BridgedOperandSet::getFunction() const { - return {set->getFunction()}; -} - //===----------------------------------------------------------------------===// // BridgedPassContext //===----------------------------------------------------------------------===// diff --git a/lib/SIL/Utils/SILBridging.cpp b/lib/SIL/Utils/SILBridging.cpp index 8f6f390f0aaf3..546edf0138ad5 100644 --- a/lib/SIL/Utils/SILBridging.cpp +++ b/lib/SIL/Utils/SILBridging.cpp @@ -545,85 +545,6 @@ BridgedInstruction BridgedBuilder::createSwitchEnumAddrInst(BridgedValue enumAdd convertCases(enumAddr.getSILValue()->getType(), enumCases, numEnumCases))}; } -//===----------------------------------------------------------------------===// -// BridgedCloner -//===----------------------------------------------------------------------===// - -// Need to put ClonerWithFixedLocation into namespace swift to forward reference -// it in SILBridging.h. -namespace swift { - -class ClonerWithFixedLocation : public SILCloner { - friend class SILInstructionVisitor; - friend class SILCloner; - - SILDebugLocation insertLoc; - -public: - ClonerWithFixedLocation(SILGlobalVariable *gVar) - : SILCloner(gVar), - insertLoc(ArtificialUnreachableLocation(), nullptr) {} - - ClonerWithFixedLocation(SILInstruction *insertionPoint) - : SILCloner(*insertionPoint->getFunction()), - insertLoc(insertionPoint->getDebugLocation()) { - Builder.setInsertionPoint(insertionPoint); - } - - SILValue getClonedValue(SILValue v) { - return getMappedValue(v); - } - - void cloneInst(SILInstruction *inst) { - visit(inst); - } - -protected: - - SILLocation remapLocation(SILLocation loc) { - return insertLoc.getLocation(); - } - - const SILDebugScope *remapScope(const SILDebugScope *DS) { - return insertLoc.getScope(); - } -}; - -} // namespace swift - -BridgedCloner::BridgedCloner(BridgedGlobalVar var, BridgedContext context) - : cloner(new ClonerWithFixedLocation(var.getGlobal())) { - context.context->notifyNewCloner(); -} - -BridgedCloner::BridgedCloner(BridgedInstruction inst, - BridgedContext context) - : cloner(new ClonerWithFixedLocation(inst.unbridged())) { - context.context->notifyNewCloner(); -} - -void BridgedCloner::destroy(BridgedContext context) { - delete cloner; - cloner = nullptr; - context.context->notifyClonerDestroyed(); -} - -BridgedValue BridgedCloner::getClonedValue(BridgedValue v) { - return {cloner->getClonedValue(v.getSILValue())}; -} - -bool BridgedCloner::isValueCloned(BridgedValue v) const { - return cloner->isValueCloned(v.getSILValue()); -} - -void BridgedCloner::clone(BridgedInstruction inst) { - cloner->cloneInst(inst.unbridged()); -} - -void BridgedCloner::recordFoldedValue(BridgedValue origValue, BridgedValue mappedValue) { - cloner->recordFoldedValue(origValue.getSILValue(), mappedValue.getSILValue()); -} - //===----------------------------------------------------------------------===// // BridgedContext //===----------------------------------------------------------------------===// From f974cc364354cb07e3110d9c3252ec53f7b73e0a Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Mon, 4 Aug 2025 11:39:36 +0100 Subject: [PATCH 24/44] Enable new licm in pass pipeline. --- lib/SILOptimizer/PassManager/PassPipeline.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/SILOptimizer/PassManager/PassPipeline.cpp b/lib/SILOptimizer/PassManager/PassPipeline.cpp index f4eed3cc5593b..01465388fcfce 100644 --- a/lib/SILOptimizer/PassManager/PassPipeline.cpp +++ b/lib/SILOptimizer/PassManager/PassPipeline.cpp @@ -381,7 +381,7 @@ void addHighLevelLoopOptPasses(SILPassPipelinePlan &P) { // before CanonicalOSSA re-hoists destroys. P.addAccessEnforcementReleaseSinking(); P.addAccessEnforcementOpts(); - P.addHighLevelLICM(); + P.addLoopInvariantCodeMotion(); // Simplify CFG after LICM that creates new exit blocks P.addSimplifyCFG(); // LICM might have added new merging potential by hoisting @@ -470,7 +470,7 @@ void addFunctionPasses(SILPassPipelinePlan &P, // late as possible before inlining because it must run between runs of the // inliner when the pipeline restarts. if (OpLevel == OptimizationLevelKind::MidLevel) { - P.addHighLevelLICM(); + P.addLoopInvariantCodeMotion(); P.addArrayCountPropagation(); P.addBoundsCheckOpts(); P.addDCE(); @@ -728,10 +728,10 @@ static void addMidLevelFunctionPipeline(SILPassPipelinePlan &P) { // A LICM pass at mid-level is mainly needed to hoist addressors of globals. // It needs to be before global_init functions are inlined. - P.addLICM(); + P.addLoopInvariantCodeMotion(); // Run loop unrolling after inlining and constant propagation, because loop // trip counts may have became constant. - P.addLICM(); + P.addLoopInvariantCodeMotion(); P.addLoopUnroll(); } @@ -837,7 +837,7 @@ static void addLateLoopOptPassPipeline(SILPassPipelinePlan &P) { // It will also set the no_nested_conflict for dynamic accesses P.addAccessEnforcementReleaseSinking(); P.addAccessEnforcementOpts(); - P.addLICM(); + P.addLoopInvariantCodeMotion(); P.addCOWOpts(); // Simplify CFG after LICM that creates new exit blocks P.addSimplifyCFG(); @@ -881,7 +881,7 @@ static void addLastChanceOptPassPipeline(SILPassPipelinePlan &P) { P.addAccessEnforcementDom(); // addAccessEnforcementDom might provide potential for LICM: // A loop might have only one dynamic access now, i.e. hoistable - P.addLICM(); + P.addLoopInvariantCodeMotion(); // Verify AccessStorage once again after optimizing and lowering OSSA. #ifndef NDEBUG From 2db57115fe17283777aa6305379adf5db5acae6f Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Mon, 4 Aug 2025 11:44:07 +0100 Subject: [PATCH 25/44] Revert back deleted vim files. --- utils/vim/ftdetect/sil.vim | 9 + utils/vim/ftdetect/swift.vim | 9 + utils/vim/ftdetect/swiftgyb.vim | 9 + utils/vim/ftplugin/swift.vim | 21 +++ utils/vim/ftplugin/swiftgyb.vim | 9 + utils/vim/syntax/sil.vim | 171 +++++++++++++++++++ utils/vim/syntax/swift.vim | 285 ++++++++++++++++++++++++++++++++ utils/vim/syntax/swiftgyb.vim | 22 +++ 8 files changed, 535 insertions(+) create mode 100644 utils/vim/ftdetect/sil.vim create mode 100644 utils/vim/ftdetect/swift.vim create mode 100644 utils/vim/ftdetect/swiftgyb.vim create mode 100644 utils/vim/ftplugin/swift.vim create mode 100644 utils/vim/ftplugin/swiftgyb.vim create mode 100644 utils/vim/syntax/sil.vim create mode 100644 utils/vim/syntax/swift.vim create mode 100644 utils/vim/syntax/swiftgyb.vim diff --git a/utils/vim/ftdetect/sil.vim b/utils/vim/ftdetect/sil.vim new file mode 100644 index 0000000000000..7e0f0e2934bc7 --- /dev/null +++ b/utils/vim/ftdetect/sil.vim @@ -0,0 +1,9 @@ +" This source file is part of the Swift.org open source project +" +" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +" Licensed under Apache License v2.0 with Runtime Library Exception +" +" See https://swift.org/LICENSE.txt for license information +" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +au BufNewFile,BufRead *.sil set ft=sil diff --git a/utils/vim/ftdetect/swift.vim b/utils/vim/ftdetect/swift.vim new file mode 100644 index 0000000000000..4812f57fe0edd --- /dev/null +++ b/utils/vim/ftdetect/swift.vim @@ -0,0 +1,9 @@ +" This source file is part of the Swift.org open source project +" +" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +" Licensed under Apache License v2.0 with Runtime Library Exception +" +" See https://swift.org/LICENSE.txt for license information +" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +au BufNewFile,BufRead *.swift set ft=swift diff --git a/utils/vim/ftdetect/swiftgyb.vim b/utils/vim/ftdetect/swiftgyb.vim new file mode 100644 index 0000000000000..3dd91226f7194 --- /dev/null +++ b/utils/vim/ftdetect/swiftgyb.vim @@ -0,0 +1,9 @@ +" This source file is part of the Swift.org open source project +" +" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +" Licensed under Apache License v2.0 with Runtime Library Exception +" +" See https://swift.org/LICENSE.txt for license information +" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +au BufNewFile,BufRead *.swift.gyb set ft=swiftgyb diff --git a/utils/vim/ftplugin/swift.vim b/utils/vim/ftplugin/swift.vim new file mode 100644 index 0000000000000..55d9e47b6398a --- /dev/null +++ b/utils/vim/ftplugin/swift.vim @@ -0,0 +1,21 @@ +" This source file is part of the Swift.org open source project +" +" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +" Licensed under Apache License v2.0 with Runtime Library Exception +" +" See https://swift.org/LICENSE.txt for license information +" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +" Only do this when not done yet for this buffer +if exists("b:did_ftplugin") + finish +endif + +let b:did_ftplugin = 1 +let b:undo_ftplugin = "setlocal comments< expandtab< tabstop< shiftwidth< smartindent<" + +setlocal comments=s1:/*,mb:*,ex:*/,:///,:// +setlocal expandtab +setlocal tabstop=2 +setlocal shiftwidth=2 +setlocal smartindent diff --git a/utils/vim/ftplugin/swiftgyb.vim b/utils/vim/ftplugin/swiftgyb.vim new file mode 100644 index 0000000000000..033b60c6729f4 --- /dev/null +++ b/utils/vim/ftplugin/swiftgyb.vim @@ -0,0 +1,9 @@ +" This source file is part of the Swift.org open source project +" +" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +" Licensed under Apache License v2.0 with Runtime Library Exception +" +" See https://swift.org/LICENSE.txt for license information +" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +runtime! ftplugin/swift.vim diff --git a/utils/vim/syntax/sil.vim b/utils/vim/syntax/sil.vim new file mode 100644 index 0000000000000..b043cd3d4ce77 --- /dev/null +++ b/utils/vim/syntax/sil.vim @@ -0,0 +1,171 @@ +" This source file is part of the Swift.org open source project +" +" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +" Licensed under Apache License v2.0 with Runtime Library Exception +" +" See https://swift.org/LICENSE.txt for license information +" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +" +" Vim syntax file +" Language: sil + +if exists("b:current_syntax") + finish +endif + +syn keyword silStage skipwhite nextgroup=silStages + \ sil_stage +syn keyword silStages + \ canonical + \ raw + +syn match silIdentifier skipwhite + \ /@\<[A-Za-z_0-9]\+\>/ + +syn match silConvention skipwhite + \ /$\?@convention/ +syn region silConvention contained contains=silConventions + \ start="@convention(" end=")" +syn keyword silConventions + \ block + \ c + \ method + \ objc_method + \ sil_differentiability_witness + \ thick + \ thin + \ witness_method + +syn match silFunctionType skipwhite + \ /@\(\\|\\|\\|\\|\\|\\|\\|\\|\\|\\|\\|\\|\\)/ +syn match silMetatypeType skipwhite + \ /@\(\\|\\|\\)/ + +" TODO: handle [tail_elems sil-type * sil-operand] +syn region silAttribute contains=silAttributes + \ start="\[" end="\]" +syn keyword silAttributes contained containedin=silAttribute + \ abort + \ deinit + \ delegatingself + \ derivedself + \ derivedselfonly + \ dynamic + \ exact + \ init + \ modify + \ mutating + \ objc + \ open + \ read + \ rootself + \ stack + \ static + \ strict + \ unknown + \ unsafe + \ var + +syn keyword swiftImport import skipwhite nextgroup=swiftImportModule +syn match swiftImportModule /\<[A-Za-z_][A-Za-z_0-9]*\>/ contained nextgroup=swiftImportComponent +syn match swiftImportComponent /\.\<[A-Za-z_][A-Za-z_0-9]*\>/ contained nextgroup=swiftImportComponent + +syn region swiftComment start="/\*" end="\*/" contains=swiftComment,swiftTodo +syn region swiftLineComment start="//" end="$" contains=swiftTodo + +syn match swiftLineComment /^#!.*/ +syn match swiftTypeName /\<[A-Z][a-zA-Z_0-9]*\>/ +syn match swiftDecimal /\<[-]\?[0-9]\+\>/ +syn match swiftDecimal /\<[-+]\?[0-9]\+\>/ + +syn match swiftTypeName /\$\*\<\?[A-Z][a-zA-Z0-9_]*\>/ +syn match swiftVarName /%\<[A-z[a-z_0-9]\+\(#[0-9]\+\)\?\>/ + +syn keyword swiftKeyword break case continue default do else for if in static switch repeat return where while skipwhite + +syn keyword swiftKeyword sil internal thunk skipwhite +syn keyword swiftKeyword public hidden private shared public_external hidden_external skipwhite +syn keyword swiftKeyword getter setter allocator initializer enumelt destroyer globalaccessor objc skipwhite +syn keyword swiftKeyword alloc_global alloc_stack alloc_ref alloc_ref_dynamic alloc_box alloc_existential_box dealloc_stack dealloc_stack_ref dealloc_box dealloc_existential_box dealloc_ref dealloc_partial_ref skipwhite +syn keyword swiftKeyword debug_value debug_value_addr skipwhite +syn keyword swiftKeyword load load_unowned store assign mark_uninitialized mark_function_escape copy_addr destroy_addr index_addr index_raw_pointer bind_memory to skipwhite +syn keyword swiftKeyword strong_retain strong_release strong_retain_unowned ref_to_unowned unowned_to_ref unowned_retain unowned_release load_weak store_unowned store_weak fix_lifetime autorelease_value set_deallocating is_unique destroy_not_escaped_closure skipwhite +syn keyword swiftKeyword function_ref integer_literal float_literal string_literal global_addr skipwhite +syn keyword swiftKeyword class_method super_method witness_method objc_method objc_super_method skipwhite +syn keyword swiftKeyword partial_apply builtin skipwhite +syn keyword swiftApplyKeyword apply try_apply skipwhite +syn keyword swiftKeyword metatype value_metatype existential_metatype skipwhite +syn keyword swiftKeyword retain_value release_value retain_value_addr release_value_addr tuple tuple_extract tuple_element_addr struct struct_extract struct_element_addr ref_element_addr skipwhite +syn keyword swiftKeyword init_enum_data_addr unchecked_enum_data unchecked_take_enum_data_addr inject_enum_addr skipwhite +syn keyword swiftKeyword init_existential_addr init_existential_value init_existential_metatype deinit_existential_addr deinit_existential_value open_existential_addr open_existential_box open_existential_box_value open_existential_metatype init_existential_ref open_existential_ref open_existential_value skipwhite +syn keyword swiftKeyword upcast address_to_pointer pointer_to_address unchecked_addr_cast unchecked_ref_cast unchecked_ref_cast_addr ref_to_raw_pointer ref_to_bridge_object ref_to_unmanaged unmanaged_to_ref raw_pointer_to_ref skipwhite +syn keyword swiftKeyword convert_function thick_to_objc_metatype objc_to_thick_metatype thin_to_thick_function unchecked_ref_bit_cast unchecked_trivial_bit_cast bridge_object_to_ref bridge_object_to_word unchecked_bitwise_cast skipwhite +syn keyword swiftKeyword objc_existential_metatype_to_object objc_metatype_to_object objc_protocol skipwhite +syn keyword swiftKeyword unconditional_checked_cast unconditional_checked_cast_addr unconditional_checked_cast_value skipwhite +syn keyword swiftKeyword cond_fail skipwhite +syn keyword swiftKeyword unreachable return throw br cond_br switch_value select_enum select_enum_addr switch_enum switch_enum_addr dynamic_method_br checked_cast_br checked_cast_value_br checked_cast_addr_br skipwhite +syn keyword swiftKeyword project_box project_existential_box project_block_storage init_block_storage_header copy_block mark_dependence skipwhite + +syn keyword swiftTypeDefinition class extension protocol struct typealias enum skipwhite nextgroup=swiftTypeName +syn region swiftTypeAttributes start="\[" end="\]" skipwhite contained nextgroup=swiftTypeName +syn match swiftTypeName /\<[A-Za-z_][A-Za-z_0-9\.]*\>/ contained nextgroup=swiftTypeParameters + +syn region swiftTypeParameters start="<" end=">" skipwhite contained + +syn keyword swiftFuncDefinition func skipwhite nextgroup=swiftFuncAttributes,swiftFuncName,swiftOperator +syn region swiftFuncAttributes start="\[" end="\]" skipwhite contained nextgroup=swiftFuncName,swiftOperator +syn match swiftFuncName /\<[A-Za-z_][A-Za-z_0-9]*\>/ skipwhite contained nextgroup=swiftTypeParameters +syn keyword swiftFuncKeyword subscript init destructor nextgroup=swiftTypeParameters + +syn keyword swiftVarDefinition var skipwhite nextgroup=swiftVarName +syn keyword swiftVarDefinition let skipwhite nextgroup=swiftVarName +syn match swiftVarName /\<[A-Za-z_][A-Za-z_0-9]*\>/ skipwhite contained + +syn keyword swiftDefinitionModifier static + +syn match swiftImplicitVarName /\$\<[A-Za-z_0-9]\+\>/ + +hi def link swiftImport Include +hi def link swiftImportModule Title +hi def link swiftImportComponent Identifier +hi def link swiftApplyKeyword Statement +hi def link swiftKeyword Statement +hi def link swiftTypeDefinition Define +hi def link swiftTypeName Type +hi def link swiftTypeParameters Special +hi def link swiftTypeAttributes PreProc +hi def link swiftFuncDefinition Define +hi def link swiftDefinitionModifier Define +hi def link swiftFuncName Function +hi def link swiftFuncAttributes PreProc +hi def link swiftFuncKeyword Function +hi def link swiftVarDefinition Define +hi def link swiftVarName Identifier +hi def link swiftImplicitVarName Identifier +hi def link swiftIdentifierKeyword Identifier +hi def link swiftTypeDeclaration Delimiter +hi def link swiftBoolean Boolean +hi def link swiftString String +hi def link swiftInterpolation Special +hi def link swiftComment Comment +hi def link swiftLineComment Comment +hi def link swiftDecimal Number +hi def link swiftHex Number +hi def link swiftOct Number +hi def link swiftBin Number +hi def link swiftOperator Function +hi def link swiftChar Character +hi def link swiftLabel Label +hi def link swiftNew Operator + +hi def link silStage Special +hi def link silStages Type +hi def link silConvention Special +hi def link silConventionParameter Special +hi def link silConventions Type +hi def link silIdentifier Identifier +hi def link silFunctionType Special +hi def link silMetatypeType Special +hi def link silAttribute PreProc + +let b:current_syntax = "sil" diff --git a/utils/vim/syntax/swift.vim b/utils/vim/syntax/swift.vim new file mode 100644 index 0000000000000..baab940550fcd --- /dev/null +++ b/utils/vim/syntax/swift.vim @@ -0,0 +1,285 @@ +" This source file is part of the Swift.org open source project +" +" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +" Licensed under Apache License v2.0 with Runtime Library Exception +" +" See https://swift.org/LICENSE.txt for license information +" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +" +" Vim syntax file +" Language: swift +" Maintainer: Joe Groff +" Last Change: 2018 Jan 21 + +if exists("b:current_syntax") + finish +endif + +syn keyword swiftKeyword + \ await + \ break + \ case + \ catch + \ continue + \ default + \ defer + \ do + \ else + \ fallthrough + \ for + \ guard + \ if + \ in + \ repeat + \ return + \ switch + \ throw + \ try + \ where + \ while +syn match swiftMultiwordKeyword + \ "indirect case" + +syn keyword swiftCoreTypes + \ Any + \ AnyObject + +syn keyword swiftImport skipwhite skipempty nextgroup=swiftImportModule + \ import + +syn keyword swiftDefinitionModifier + \ async + \ convenience + \ dynamic + \ fileprivate + \ final + \ internal + \ lazy + \ nonmutating + \ open + \ override + \ prefix + \ private + \ public + \ reasync + \ required + \ rethrows + \ static + \ throws + \ weak + +syn keyword swiftInOutKeyword skipwhite skipempty nextgroup=swiftTypeName + \ inout + +syn keyword swiftIdentifierKeyword + \ Self + \ metatype + \ self + \ super + +syn keyword swiftFuncKeywordGeneral skipwhite skipempty nextgroup=swiftTypeParameters + \ init + +syn keyword swiftFuncKeyword + \ deinit + \ subscript + +syn keyword swiftScope + \ autoreleasepool + +syn keyword swiftMutating skipwhite skipempty nextgroup=swiftFuncDefinition + \ mutating +syn keyword swiftFuncDefinition skipwhite skipempty nextgroup=swiftTypeName,swiftOperator + \ func + +syn keyword swiftTypeDefinition skipwhite skipempty nextgroup=swiftTypeName + \ class + \ enum + \ extension + \ operator + \ precedencegroup + \ protocol + \ struct + +syn keyword swiftTypeAliasDefinition skipwhite skipempty nextgroup=swiftTypeAliasName + \ associatedtype + \ typealias + +syn match swiftMultiwordTypeDefinition skipwhite skipempty nextgroup=swiftTypeName + \ "indirect enum" + +syn keyword swiftVarDefinition skipwhite skipempty nextgroup=swiftVarName + \ let + \ var + +syn keyword swiftLabel + \ get + \ set + \ didSet + \ willSet + +syn keyword swiftBoolean + \ false + \ true + +syn keyword swiftNil + \ nil + +syn match swiftImportModule contained nextgroup=swiftImportComponent + \ /\<[A-Za-z_][A-Za-z_0-9]*\>/ +syn match swiftImportComponent contained nextgroup=swiftImportComponent + \ /\.\<[A-Za-z_][A-Za-z_0-9]*\>/ + +syn match swiftTypeAliasName contained skipwhite skipempty nextgroup=swiftTypeAliasValue + \ /\<[A-Za-z_][A-Za-z_0-9]*\>/ +syn match swiftTypeName contained skipwhite skipempty nextgroup=swiftTypeParameters + \ /\<[A-Za-z_][A-Za-z_0-9\.]*\>/ +syn match swiftVarName contained skipwhite skipempty nextgroup=swiftTypeDeclaration + \ /\<[A-Za-z_][A-Za-z_0-9]*\>/ +syn match swiftImplicitVarName + \ /\$\<[A-Za-z_0-9]\+\>/ + +" TypeName[Optionality]? +syn match swiftType contained skipwhite skipempty nextgroup=swiftTypeParameters + \ /\<[A-Za-z_][A-Za-z_0-9\.]*\>[!?]\?/ +" [Type:Type] (dictionary) or [Type] (array) +syn region swiftType contained contains=swiftTypePair,swiftType + \ matchgroup=Delimiter start=/\[/ end=/\]/ +syn match swiftTypePair contained skipwhite skipempty nextgroup=swiftTypeParameters,swiftTypeDeclaration + \ /\<[A-Za-z_][A-Za-z_0-9\.]*\>[!?]\?/ +" (Type[, Type]) (tuple) +" FIXME: we should be able to use skip="," and drop swiftParamDelim +syn region swiftType contained contains=swiftType,swiftParamDelim + \ matchgroup=Delimiter start="[^@]\?(" end=")" matchgroup=NONE skip="," +syn match swiftParamDelim contained + \ /,/ +" (generics) +syn region swiftTypeParameters contained contains=swiftVarName,swiftConstraint + \ matchgroup=Delimiter start="<" end=">" matchgroup=NONE skip="," +syn keyword swiftConstraint contained + \ where + +syn match swiftTypeAliasValue skipwhite skipempty nextgroup=swiftType + \ /=/ +syn match swiftTypeDeclaration skipwhite skipempty nextgroup=swiftType,swiftInOutKeyword + \ /:/ +syn match swiftTypeDeclaration skipwhite skipempty nextgroup=swiftType + \ /->/ + +syn match swiftKeyword + \ /\/ +syn region swiftCaseLabelRegion + \ matchgroup=swiftKeyword start=/\/ matchgroup=Delimiter end=/:/ oneline contains=TOP +syn region swiftDefaultLabelRegion + \ matchgroup=swiftKeyword start=/\/ matchgroup=Delimiter end=/:/ oneline + +syn region swiftParenthesisRegion contains=TOP + \ matchgroup=NONE start=/(/ end=/)/ + +syn region swiftString contains=swiftInterpolationRegion + \ start=/"/ skip=/\\\\\|\\"/ end=/"/ +syn region swiftInterpolationRegion contained contains=TOP + \ matchgroup=swiftInterpolation start=/\\(/ end=/)/ +syn region swiftComment contains=swiftComment,swiftTodo + \ start="/\*" end="\*/" +syn region swiftLineComment contains=swiftTodo + \ start="//" end="$" + +syn match swiftDecimal + \ /[+\-]\?\<\([0-9][0-9_]*\)\([.][0-9_]*\)\?\([eE][+\-]\?[0-9][0-9_]*\)\?\>/ +syn match swiftHex + \ /[+\-]\?\<0x[0-9A-Fa-f][0-9A-Fa-f_]*\(\([.][0-9A-Fa-f_]*\)\?[pP][+\-]\?[0-9][0-9_]*\)\?\>/ +syn match swiftOct + \ /[+\-]\?\<0o[0-7][0-7_]*\>/ +syn match swiftBin + \ /[+\-]\?\<0b[01][01_]*\>/ + +syn match swiftOperator skipwhite skipempty nextgroup=swiftTypeParameters + \ "\.\@!&|^~]\@!&|^~]*\|*/\@![/=\-+*%<>!&|^~]*\|->\@![/=\-+*%<>!&|^~]*\|[=+%<>!&|^~][/=\-+*%<>!&|^~]*\)" +syn match swiftOperator skipwhite skipempty nextgroup=swiftTypeParameters + \ "\.\.[<.]" + +syn match swiftChar + \ /'\([^'\\]\|\\\(["'tnr0\\]\|x[0-9a-fA-F]\{2}\|u[0-9a-fA-F]\{4}\|U[0-9a-fA-F]\{8}\)\)'/ + +syn match swiftTupleIndexNumber contains=swiftDecimal + \ /\.[0-9]\+/ +syn match swiftDecimal contained + \ /[0-9]\+/ + +" This is a superset of the Preproc macros below, so it must come FIRST +syn match swiftFreestandingMacro + \ /#\<[A-Za-z_][A-Za-z_0-9]*\>/ +syn match swiftPreproc + \ /#\(\\|\\|\\|\\|\\)/ +syn match swiftPreproc + \ /^\s*#\(\\|\\|\\|\\|\\|\\)/ +syn region swiftPreprocFalse + \ start="^\s*#\\s\+\" end="^\s*#\(\\|\\|\\)" + +syn match swiftAttribute + \ /@\<\w\+\>/ skipwhite skipempty nextgroup=swiftType,swiftTypeDefinition + +syn keyword swiftTodo MARK TODO FIXME contained + +syn match swiftCastOp skipwhite skipempty nextgroup=swiftType,swiftCoreTypes + \ "\" +syn match swiftCastOp skipwhite skipempty nextgroup=swiftType,swiftCoreTypes + \ "\[!?]\?" + +syn match swiftNilOps + \ "??" + +syn region swiftReservedIdentifier oneline + \ start=/`/ end=/`/ + +hi def link swiftImport Include +hi def link swiftImportModule Title +hi def link swiftImportComponent Identifier +hi def link swiftKeyword Statement +hi def link swiftCoreTypes Type +hi def link swiftMultiwordKeyword Statement +hi def link swiftTypeDefinition Define +hi def link swiftMultiwordTypeDefinition Define +hi def link swiftType Type +hi def link swiftTypePair Type +hi def link swiftTypeAliasName Identifier +hi def link swiftTypeName Function +hi def link swiftConstraint Special +hi def link swiftFuncDefinition Define +hi def link swiftDefinitionModifier Operator +hi def link swiftInOutKeyword Define +hi def link swiftFuncKeyword Function +hi def link swiftFuncKeywordGeneral Function +hi def link swiftTypeAliasDefinition Define +hi def link swiftVarDefinition Define +hi def link swiftVarName Identifier +hi def link swiftImplicitVarName Identifier +hi def link swiftIdentifierKeyword Identifier +hi def link swiftTypeAliasValue Delimiter +hi def link swiftTypeDeclaration Delimiter +hi def link swiftTypeParameters Delimiter +hi def link swiftBoolean Boolean +hi def link swiftString String +hi def link swiftInterpolation Special +hi def link swiftComment Comment +hi def link swiftLineComment Comment +hi def link swiftDecimal Number +hi def link swiftHex Number +hi def link swiftOct Number +hi def link swiftBin Number +hi def link swiftOperator Function +hi def link swiftChar Character +hi def link swiftLabel Operator +hi def link swiftMutating Statement +hi def link swiftPreproc PreCondit +hi def link swiftPreprocFalse Comment +hi def link swiftFreestandingMacro Macro +hi def link swiftAttribute Type +hi def link swiftTodo Todo +hi def link swiftNil Constant +hi def link swiftCastOp Operator +hi def link swiftNilOps Operator +hi def link swiftScope PreProc + +let b:current_syntax = "swift" diff --git a/utils/vim/syntax/swiftgyb.vim b/utils/vim/syntax/swiftgyb.vim new file mode 100644 index 0000000000000..300da06dd3726 --- /dev/null +++ b/utils/vim/syntax/swiftgyb.vim @@ -0,0 +1,22 @@ +" This source file is part of the Swift.org open source project +" +" Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +" Licensed under Apache License v2.0 with Runtime Library Exception +" +" See https://swift.org/LICENSE.txt for license information +" See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +" +" Vim syntax file +" Language: gyb on swift + +runtime! syntax/swift.vim +unlet b:current_syntax + +syn include @Python syntax/python.vim +syn region pythonCode matchgroup=gybPythonCode start=+^ *%+ end=+$+ contains=@Python keepend +syn region pythonCode matchgroup=gybPythonCode start=+%{+ end=+}%+ contains=@Python keepend +syn match gybPythonCode /\${[^}]*}/ +hi def link gybPythonCode CursorLineNr + +let b:current_syntax = "swiftgyb" + From 448e7f7707cd957c4b0cdcefe3042fb6076b1372 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Tue, 5 Aug 2025 15:34:17 +0100 Subject: [PATCH 26/44] Add simple formatting suggested changes. --- .../Optimizer/Analysis/DominatorTree.swift | 16 +- .../Sources/Optimizer/Analysis/LoopTree.swift | 105 ++--- .../LoopInvariantCodeMotion.swift | 370 ++++++------------ .../Optimizer/PassManager/ContextCommon.swift | 15 + .../Sources/Optimizer/Utilities/Cloner.swift | 18 +- .../Optimizer/Utilities/OptUtils.swift | 3 +- .../Sources/SIL/Utilities/AccessUtils.swift | 4 + .../swift/SILOptimizer/OptimizerBridging.h | 1 + lib/SILOptimizer/Utils/OptimizerBridging.cpp | 4 + .../default-arguments-multifile.swift | 6 +- 10 files changed, 193 insertions(+), 349 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift index 7089a237ba07c..132434c5eb028 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift @@ -22,23 +22,23 @@ struct DominatorTree { } struct DomChildren: BridgedRandomAccessCollection { - let bridgedDomTree: BridgedDomTree - let bb: BasicBlock + private let bridgedDomTree: BridgedDomTree + private let block: BasicBlock - public let count: Int + let count: Int - public var startIndex: Int { return 0 } - public var endIndex: Int { return count } + var startIndex: Int { return 0 } + var endIndex: Int { return count } init(bridgedDomTree: BridgedDomTree, bb: BasicBlock) { self.bridgedDomTree = bridgedDomTree - self.bb = bb + self.block = bb self.count = bridgedDomTree.getNumberOfChildren(bb.bridged) } - public subscript(_ index: Int) -> BasicBlock { + subscript(_ index: Int) -> BasicBlock { assert(index >= startIndex && index < endIndex) - return bridgedDomTree.getChildAt(bb.bridged, index).block + return bridgedDomTree.getChildAt(block.bridged, index).block } } diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index 7eb59f858802e..b96bfdcdfab34 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -15,50 +15,34 @@ import OptimizerBridging /// Describes top level loops. struct LoopTree { - private let context: Context private let bridged: BridgedLoopTree let loops: TopLevelLoopArray - init(bridged: BridgedLoopTree, context: Context) { - self.context = context + init(bridged: BridgedLoopTree, context: FunctionPassContext) { self.bridged = bridged - self.loops = TopLevelLoopArray(bridged, context: context) + self.loops = TopLevelLoopArray(bridged) } - func splitCriticalEdge( - basicBlock: BasicBlock, - edgeIndex: Int, - domTree: DominatorTree - ) -> BasicBlock? { + func splitCriticalEdge(basicBlock: BasicBlock, edgeIndex: Int, domTree: DominatorTree) -> BasicBlock? { guard basicBlock.isCriticalEdge(edgeIndex: edgeIndex) else { return nil } - return splitEdge( - basicBlock: basicBlock, - edgeIndex: edgeIndex, - domTree: domTree - ) + return splitEdge(basicBlock: basicBlock, edgeIndex: edgeIndex, domTree: domTree) } - func splitEdge( - basicBlock: BasicBlock, - edgeIndex: Int, - domTree: DominatorTree - ) -> BasicBlock? { + func splitEdge(basicBlock: BasicBlock, edgeIndex: Int, domTree: DominatorTree) -> BasicBlock? { return bridged.splitEdge(basicBlock.bridged, edgeIndex, domTree.bridged).block } } -/// Describes a loop with it's children. +/// Describes a loop with its children. struct Loop { - private let context: Context private let bridged: BridgedLoop let innerLoops: LoopArray - let basicBlocks: BasicBlockArray -// var basicBlockSet: BasicBlockSet + let loopBlocks: LoopBlocks var preheader: BasicBlock? { bridged.getPreheader().block @@ -71,22 +55,22 @@ struct Loop { var exitingAndLatchBlocks: some Sequence { return header.predecessors.lazy .filter { predecessor in - basicBlocks.contains(predecessor) && !isLoopExiting(bb: predecessor) + loopBlocks.contains(predecessor) && !isLoopExiting(loopBlock: predecessor) } + exitingBlocks } var exitBlocks: some Sequence { - return basicBlocks.lazy + return loopBlocks.lazy .flatMap(\.successors) .filter { succesor in - !basicBlocks.contains(succesor) + !loopBlocks.contains(succesor) } } var exitingBlocks: some Sequence { - return basicBlocks.lazy + return loopBlocks.lazy .filter { bb in - isLoopExiting(bb: bb) + isLoopExiting(loopBlock: bb) } } @@ -98,20 +82,12 @@ struct Loop { return exitBlocks.isEmpty } - private func isLoopExiting(bb: BasicBlock) -> Bool { - return bb.successors - .contains { succesor in - !basicBlocks.contains(succesor) - } + private func isLoopExiting(loopBlock: BasicBlock) -> Bool { + return loopBlock.successors.contains { !loopBlocks.contains($0) } } - func getBlocksThatDominateAllExitingAndLatchBlocks( - domTree: DominatorTree - ) -> some Sequence { - return getBlocksThatDominateAllExitingAndLatchBlocksHelper( - bb: header, - domTree: domTree - ) + func getBlocksThatDominateAllExitingAndLatchBlocks(domTree: DominatorTree) -> some Sequence { + return getBlocksThatDominateAllExitingAndLatchBlocksHelper(bb: header, domTree: domTree) } private func getBlocksThatDominateAllExitingAndLatchBlocksHelper( @@ -133,70 +109,65 @@ struct Loop { } } - init(bridged: BridgedLoop, context: Context) { - self.context = context + init(bridged: BridgedLoop) { self.bridged = bridged - self.innerLoops = LoopArray(bridged, context: context) - self.basicBlocks = BasicBlockArray(bridged) + self.innerLoops = LoopArray(bridged) + self.loopBlocks = LoopBlocks(bridged) } } struct TopLevelLoopArray: BridgedRandomAccessCollection { - private let context: Context private let bridgedLoopTree: BridgedLoopTree - public let count: Int + let count: Int - public var startIndex: Int { return 0 } - public var endIndex: Int { return count } + var startIndex: Int { return 0 } + var endIndex: Int { return count } - public init(_ bridgedLoopTree: BridgedLoopTree, context: Context) { - self.context = context + init(_ bridgedLoopTree: BridgedLoopTree) { self.bridgedLoopTree = bridgedLoopTree self.count = bridgedLoopTree.getTopLevelLoopCount() } - public subscript(_ index: Int) -> Loop { + subscript(_ index: Int) -> Loop { assert(index >= startIndex && index < endIndex) - return Loop(bridged: bridgedLoopTree.getLoop(index), context: context) + return Loop(bridged: bridgedLoopTree.getLoop(index)) } } struct LoopArray: BridgedRandomAccessCollection { - private let context: Context private let bridgedLoop: BridgedLoop - public let count: Int + let count: Int - public var startIndex: Int { return 0 } - public var endIndex: Int { return count } + var startIndex: Int { return 0 } + var endIndex: Int { return count } - public init(_ bridgedLoop: BridgedLoop, context: Context) { - self.context = context + init(_ bridgedLoop: BridgedLoop) { self.bridgedLoop = bridgedLoop self.count = bridgedLoop.getInnerLoopCount() } - public subscript(_ index: Int) -> Loop { + subscript(_ index: Int) -> Loop { assert(index >= startIndex && index < endIndex) - return Loop(bridged: bridgedLoop.getInnerLoop(index), context: context) + return Loop(bridged: bridgedLoop.getInnerLoop(index)) } } -struct BasicBlockArray: BridgedRandomAccessCollection { +struct LoopBlocks: BridgedRandomAccessCollection { private let bridgedLoop: BridgedLoop - public let count: Int + let count: Int - public var startIndex: Int { return 0 } - public var endIndex: Int { return count } + var startIndex: Int { return 0 } + var endIndex: Int { return count } - public init(_ bridgedLoop: BridgedLoop) { + init(_ bridgedLoop: BridgedLoop) { self.bridgedLoop = bridgedLoop self.count = bridgedLoop.getBasicBlockCount() } - public subscript(_ index: Int) -> BasicBlock { + subscript(_ index: Int) -> BasicBlock { assert(index >= startIndex && index < endIndex) return bridgedLoop.getBasicBlock(index).block } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 4905fed93041b..5799c24d5b7f6 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -13,47 +13,28 @@ import SIL import OptimizerBridging -let loopInvariantCodeMotionPass = FunctionPass( - name: "loop-invariant-code-motion" -) { function, context in +let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion") { function, context in for loop in context.loopTree.loops { - optimizeTopLevelLoop( - topLevelLoop: loop, - context: context - ) + optimizeTopLevelLoop(topLevelLoop: loop, context) } } -private func optimizeTopLevelLoop( - topLevelLoop: Loop, - context: FunctionPassContext -) { - var workList = getWorkList(topLevelLoop: topLevelLoop, context: context) - defer { workList.deinitialize() } +private func optimizeTopLevelLoop(topLevelLoop: Loop, _ context: FunctionPassContext) { + var innerLoops = getWorkList(topLevelLoop: topLevelLoop, context) + defer { innerLoops.deinitialize() } - while let thisLoop = workList.pop() { + while let thisLoop = innerLoops.pop() { var thisLoopChanged = false repeat { - guard - var movableInstructions = analyzeLoop( - loop: thisLoop, - context: context - ) - else { - return // Encountered a loop without preheader. Return early. + if var movableInstructions = analyzeLoopAndSplitLoads(loop: thisLoop, context) { + thisLoopChanged = optimizeLoop(loop: thisLoop, movableInstructions: &movableInstructions, context) } - - thisLoopChanged = optimizeLoop( - loop: thisLoop, - movableInstructions: &movableInstructions, - context: context - ) } while thisLoopChanged } } -private func getWorkList(topLevelLoop: Loop, context: Context) -> Stack { +private func getWorkList(topLevelLoop: Loop, _ context: Context) -> Stack { var tmp1 = Stack(context) var tmp2 = Stack(context) var workList = Stack(context) @@ -103,6 +84,8 @@ struct AnalyzedInstructions { private(set) var loopSideEffectCount = 0 private(set) var loadsCount = 0 + private(set) var hasOtherMemReadingInsts = false + init (_ context: FunctionPassContext) { self.globalInitCalls = Stack(context) self.loopSideEffects = Stack(context) @@ -164,10 +147,7 @@ struct AnalyzedInstructions { } } -private func analyzeLoop( - loop: Loop, - context: FunctionPassContext -) -> MovableInstructions? { +private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext) -> MovableInstructions? { guard let preheader = loop.preheader else { return nil } @@ -176,19 +156,13 @@ private func analyzeLoop( var analyzedInstructions = AnalyzedInstructions(context) defer { analyzedInstructions.deinitialize() } - var hasOtherMemReadingInsts = false - - for bb in loop.basicBlocks { + for bb in loop.loopBlocks { var blockSideEffects = Stack(context) defer { blockSideEffects.deinitialize() } for inst in bb.instructions { if inst.hasOwnershipOperandsOrResults { - inst.analyzeSideEffects( - analyzedInstructions: &analyzedInstructions, - blockSideEffects: &blockSideEffects, - hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) + analyzedInstructions.analyzeSideEffects(ofInst: inst, blockSideEffects: &blockSideEffects) if let fullApply = inst as? FullApplySite { analyzedInstructions.append(fullApply) @@ -212,27 +186,15 @@ private func analyzeLoop( } analyzedInstructions.append(storeInst) movableInstructions.loadsAndStores.append(storeInst) - storeInst.analyzeSideEffects( - analyzedInstructions: &analyzedInstructions, - blockSideEffects: &blockSideEffects, - hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) + analyzedInstructions.analyzeSideEffects(ofInst: storeInst, blockSideEffects: &blockSideEffects) case let beginAccessInst as BeginAccessInst: analyzedInstructions.append(beginAccessInst) - beginAccessInst.analyzeSideEffects( - analyzedInstructions: &analyzedInstructions, - blockSideEffects: &blockSideEffects, - hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) + analyzedInstructions.analyzeSideEffects(ofInst: beginAccessInst, blockSideEffects: &blockSideEffects) case let refElementAddrInst as RefElementAddrInst: movableInstructions.simplyHoistableInsts.append(refElementAddrInst) case let condFailInst as CondFailInst: movableInstructions.hoistUp.append(condFailInst) - condFailInst.analyzeSideEffects( - analyzedInstructions: &analyzedInstructions, - blockSideEffects: &blockSideEffects, - hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) + analyzedInstructions.analyzeSideEffects(ofInst: condFailInst, blockSideEffects: &blockSideEffects) case let applyInst as ApplyInst: if applyInst.isSafeReadOnlyApply( calleeAnalysis: context.calleeAnalysis @@ -266,16 +228,9 @@ private func analyzeLoop( default: break } - inst.analyzeSideEffects( - analyzedInstructions: &analyzedInstructions, - blockSideEffects: &blockSideEffects, - hasOtherMemReadingInsts: &hasOtherMemReadingInsts - ) + analyzedInstructions.analyzeSideEffects(ofInst: inst, blockSideEffects: &blockSideEffects) - if inst.canBeHoisted( - outOf: loop, - context: context - ) { + if inst.canBeHoisted(outOf: loop, context) { movableInstructions.hoistUp.append(inst) } } @@ -298,29 +253,24 @@ private func analyzeLoop( // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. if analyzedInstructions.loadsCount * analyzedInstructions.loopSideEffectCount < 8000 { for load in analyzedInstructions.loads { - if !load.mayWriteTo( - sideEffects: analyzedInstructions.loopSideEffects, - aliasAnalysis: context.aliasAnalysis - ) { + if !load.mayWriteTo(sideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis) { movableInstructions.hoistUp.append(load) } } } - if !analyzedInstructions.globalInitCalls.isEmpty { - for globalInitCall in analyzedInstructions.globalInitCalls { - if !globalInitCall.globalInitMayConflictWith( - preheader: preheader, - sideEffects: analyzedInstructions.loopSideEffects, - aliasAnalysis: context.aliasAnalysis, - postDomTree: context.postDominatorTree - ) { - movableInstructions.hoistUp.append(globalInitCall) - } + for globalInitCall in analyzedInstructions.globalInitCalls { + if !globalInitCall.globalInitMayConflictWith( + preheader: preheader, + sideEffects: analyzedInstructions.loopSideEffects, + aliasAnalysis: context.aliasAnalysis, + postDomTree: context.postDominatorTree + ) { + movableInstructions.hoistUp.append(globalInitCall) } } - if !hasOtherMemReadingInsts { + if !analyzedInstructions.hasOtherMemReadingInsts { for storeInst in analyzedInstructions.stores { let accessPath = storeInst.destination.accessPath if accessPath.isLoopInvariant(loop: loop), @@ -330,11 +280,11 @@ private func analyzeLoop( aliasAnalysis: context.aliasAnalysis ), !movableInstructions.loadAndStoreAddrs.contains(accessPath), - movableInstructions.splitLoads( - analyzedInstructions: &analyzedInstructions, + analyzedInstructions.splitLoads( + movableInstructions: &movableInstructions, storeAddr: storeInst.destination, accessPath: accessPath, - context: context + context ) { movableInstructions.loadAndStoreAddrs.append(accessPath) } @@ -373,153 +323,111 @@ private func analyzeLoop( private func optimizeLoop( loop: Loop, movableInstructions: inout MovableInstructions, - context: FunctionPassContext + _ context: FunctionPassContext ) -> Bool { var changed = false // TODO: If we hoist tuple_element_addr and struct_element_addr instructions here, hoistAndSinkLoadsAndStores could converge after just one execution! - changed = movableInstructions.simpleHoistInstructions( - loop: loop, - context: context - ) || changed - - changed = movableInstructions.hoistAndSinkLoadsAndStores( - outOf: loop, - context: context - ) || changed - - changed = movableInstructions.hoistInstructions( - loop: loop, - context: context - ) || changed - - changed = movableInstructions.sinkInstructions( - loop: loop, - context: context - ) || changed - - changed = movableInstructions.hoistWithSinkScopedInstructions( - loop: loop, - context: context - ) || changed + changed = movableInstructions.simpleHoistInstructions(loop: loop, context) || changed + changed = movableInstructions.hoistAndSinkLoadsAndStores(outOf: loop, context) || changed + changed = movableInstructions.hoistInstructions(loop: loop, context) || changed + changed = movableInstructions.sinkInstructions(loop: loop, context) || changed + changed = movableInstructions.hoistWithSinkScopedInstructions(loop: loop, context) || changed return changed } -extension MovableInstructions { +extension AnalyzedInstructions { + mutating func analyzeSideEffects( + ofInst inst: Instruction, + blockSideEffects: inout Stack + ) { + if inst.mayHaveSideEffects { + appendSideEffect(inst) + blockSideEffects.append(inst) + } else if inst.mayReadFromMemory { + hasOtherMemReadingInsts = true + } + } + mutating func splitLoads( - analyzedInstructions: inout AnalyzedInstructions, + movableInstructions: inout MovableInstructions, storeAddr: Value, accessPath: AccessPath, - context: FunctionPassContext + _ context: FunctionPassContext ) -> Bool { - var tmpLoads = Stack(context) + var newLoads = Stack(context) defer { - analyzedInstructions.append(newLoads: tmpLoads) - tmpLoads.deinitialize() + append(newLoads: newLoads) + newLoads.deinitialize() } var splitCounter = 0 - while let loadInst = analyzedInstructions.popLoad() { + while let loadInst = popLoad() { guard splitCounter <= 6 else { - tmpLoads.push(loadInst) + newLoads.push(loadInst) return false } - guard !loadInst.isDeleted, - loadInst.mayRead(fromAddress: storeAddr, context.aliasAnalysis), - !accessPath.isEqualOrContains(loadInst.operand.value.accessPath) - else { - tmpLoads.push(loadInst) - continue - } - - let loadAccessPath = loadInst.operand.value.accessPath - guard !accessPath.isEqualOrContains(loadAccessPath) else { - tmpLoads.push(loadInst) + guard !loadInst.isDeleted, loadInst.operand.value.accessPath.contains(accessPath) else { + newLoads.push(loadInst) continue } if let splitLoads = loadInst.trySplit(alongPath: accessPath.projectionPath, context) { splitCounter += splitLoads.count - loadsAndStores.replace([loadInst], with: splitLoads) - tmpLoads.append(contentsOf: splitLoads) + movableInstructions.loadsAndStores.replace([loadInst], with: splitLoads) + newLoads.append(contentsOf: splitLoads) } } return true } - - mutating func simpleHoistInstructions( - loop: Loop, - context: FunctionPassContext - ) -> Bool { +} + +extension MovableInstructions { + mutating func simpleHoistInstructions(loop: Loop, _ context: FunctionPassContext) -> Bool { var changed = false for inst in simplyHoistableInsts { - changed = inst.hoist( - outOf: loop, - context: context - ) || changed + changed = inst.hoist(outOf: loop, context) || changed } return changed } - mutating func hoistAndSinkLoadsAndStores( - outOf loop: Loop, - context: FunctionPassContext - ) -> Bool { + mutating func hoistAndSinkLoadsAndStores(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { var changed = false for accessPath in loadAndStoreAddrs { - changed = hoistAndSinkLoadAndStore( - outOf: loop, - accessPath: accessPath, - context: context - ) || changed + changed = hoistAndSinkLoadAndStore(outOf: loop, accessPath: accessPath, context: context) || changed } return changed } - mutating func hoistInstructions( - loop: Loop, - context: FunctionPassContext - ) -> Bool { + mutating func hoistInstructions(loop: Loop, _ context: FunctionPassContext) -> Bool { let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(domTree: context.dominatorTree) var changed = false for bb in dominatingBlocks { for inst in bb.instructions where hoistUp.contains(inst) { - changed = inst.hoist( - outOf: loop, - context: context - ) || changed + changed = inst.hoist(outOf: loop, context) || changed } } return changed } - mutating func sinkInstructions( - loop: Loop, - context: FunctionPassContext - ) -> Bool { + mutating func sinkInstructions(loop: Loop, _ context: FunctionPassContext) -> Bool { let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(domTree: context.dominatorTree) var changed = false for inst in sinkDown where dominatingBlocks.contains(inst.parentBlock) { - changed = inst.sink( - outOf: loop, - context: context - ) || changed + changed = inst.sink(outOf: loop, context) || changed } return changed } - mutating func hoistWithSinkScopedInstructions( - loop: Loop, - context: FunctionPassContext - ) -> Bool { + mutating func hoistWithSinkScopedInstructions(loop: Loop, _ context: FunctionPassContext) -> Bool { guard !loop.hasNoExitBlocks else { return false } @@ -531,12 +439,12 @@ extension MovableInstructions { continue } - guard specialInst.hoist(outOf: loop, context: context) else { + guard specialInst.hoist(outOf: loop, context) else { continue } for endAccess in beginAccessInst.endAccessInstructions { - endAccess.sink(outOf: loop, context: context) + endAccess.sink(outOf: loop, context) } changed = true @@ -552,15 +460,12 @@ extension MovableInstructions { ) -> Bool { let exitingAndLatchBlocks = loop.exitingAndLatchBlocks - guard loop.storesCommonlyDominateExits( - accessPath: accessPath, - context: context - ) else { + guard loop.storesCommonlyDominateExits(accessPath: accessPath, context) else { return false } for exitingOrLatchBlock in exitingAndLatchBlocks { - for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !loop.basicBlocks.contains(succesor) { + for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !loop.loopBlocks.contains(succesor) { _ = context.loopTree.splitCriticalEdge( basicBlock: exitingOrLatchBlock.terminator.parentBlock, edgeIndex: index, @@ -649,7 +554,7 @@ extension MovableInstructions { rootVal: rootVal, rootAccessPath: accessPath, beforeInst: loadInst, - context: context + context ) else { continue } @@ -659,7 +564,7 @@ extension MovableInstructions { } for exitingOrLatchBlock in exitingAndLatchBlocks { - for succesor in exitingOrLatchBlock.successors where !loop.basicBlocks.contains(succesor) { + for succesor in exitingOrLatchBlock.successors where !loop.loopBlocks.contains(succesor) { guard let firstInst = succesor.instructions.first else { continue } @@ -685,10 +590,7 @@ extension MovableInstructions { } extension Loop { - fileprivate func storesCommonlyDominateExits( - accessPath: AccessPath, - context: FunctionPassContext - ) -> Bool { + fileprivate func storesCommonlyDominateExits(accessPath: AccessPath, _ context: FunctionPassContext) -> Bool { var stores = BasicBlockSet(context) var storesNotAlive = BasicBlockSet(context) var uses = Stack(context) @@ -718,7 +620,7 @@ extension Loop { var changed = false repeat { changed = false - for block in basicBlocks where !storesNotAlive.contains(block) && !stores.contains(block) && block.predecessors.contains(where: { storesNotAlive.contains($0) }) { + for block in loopBlocks where !storesNotAlive.contains(block) && !stores.contains(block) && block.predecessors.contains(where: { storesNotAlive.contains($0) }) { storesNotAlive.insert(block) changed = true } @@ -754,19 +656,6 @@ extension Instruction { } } - fileprivate func analyzeSideEffects( - analyzedInstructions: inout AnalyzedInstructions, - blockSideEffects: inout Stack, - hasOtherMemReadingInsts: inout Bool - ) { - if mayHaveSideEffects { - analyzedInstructions.appendSideEffect(self) - blockSideEffects.append(self) - } else if mayReadFromMemory { - hasOtherMemReadingInsts = true - } - } - fileprivate var hasOwnershipOperandsOrResults: Bool { guard parentFunction.hasOwnership else { return false } @@ -774,25 +663,25 @@ extension Instruction { || operands.contains(where: { $0.value.ownership != .none }) } - fileprivate func canBeHoisted( - outOf loop: Loop, - context: FunctionPassContext - ) -> Bool { + fileprivate func canBeHoisted(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { guard let preheader = loop.preheader else { return false } - if self is TermInst || self is Allocation || self is Deallocation { + switch self { + case is TermInst, is Allocation, is Deallocation: return false - } - - switch getArraySemanticsCallKind() { - case .getCount, .getCapacity: - if canHoistArraySemanticsCall(to: preheader.terminator, context) { - return true + case is ApplyInst: + switch arraySemanticsCallKind { + case .getCount, .getCapacity: + if canHoistArraySemanticsCall(to: preheader.terminator, context) { + return true + } + case .arrayPropsIsNativeTypeChecked: + return false + default: + break } - case .arrayPropsIsNativeTypeChecked: - return false default: break } @@ -805,12 +694,9 @@ extension Instruction { } @discardableResult - fileprivate func hoist( - outOf loop: Loop, - context: FunctionPassContext - ) -> Bool { + fileprivate func hoist(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { guard let preheader = loop.preheader, - operands.allSatisfy({ !loop.basicBlocks.contains($0.value.parentBlock) }) else { + operands.allSatisfy({ !loop.loopBlocks.contains($0.value.parentBlock) }) else { return false } @@ -836,10 +722,7 @@ extension Instruction { } @discardableResult - fileprivate func sink( - outOf loop: Loop, - context: FunctionPassContext - ) -> Bool { + fileprivate func sink(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { var isSingleExit = loop.isSingleExit let exitBlocks = loop.exitBlocks let exitingBlocks = loop.exitingBlocks @@ -930,19 +813,6 @@ extension Instruction { ) } } - - func getArraySemanticsCallKind() -> BridgedArrayCallKind { - return BridgedPassContext.getArraySemanticsCallKind(self.bridged) - } - - func canHoistArraySemanticsCall(to toInst: Instruction, _ context: FunctionPassContext) -> Bool { - return context.bridgedPassContext.canHoistArraySemanticsCall(self.bridged, toInst.bridged) - } - - func hoistArraySemanticsCall(before toInst: Instruction, _ context: some MutatingContext) { - context.bridgedPassContext.hoistArraySemanticsCall(self.bridged, toInst.bridged) // Internally updates dom tree. - context.notifyInstructionsChanged() - } } extension UnaryInstruction { @@ -972,7 +842,7 @@ extension LoadInst { rootVal: Value, rootAccessPath: AccessPath, beforeInst: Instruction, - context: FunctionPassContext + _ context: FunctionPassContext ) -> Bool { guard operand.value.accessPath != rootAccessPath else { replace(with: rootVal, context) @@ -1007,15 +877,11 @@ extension LoadInst { return true } - fileprivate func accesses( - _ accessPath: AccessPath - ) -> Bool { + fileprivate func accesses(_ accessPath: AccessPath) -> Bool { return accessPath.getProjection(to: self.address.accessPath)?.isMaterializable ?? false } - fileprivate func overlaps( - accessPath: AccessPath - ) -> Bool { + fileprivate func overlaps(accessPath: AccessPath) -> Bool { guard self.loadOwnership != .take else { // TODO: handle LoadOwnershipQualifier::Take return false } @@ -1025,9 +891,7 @@ extension LoadInst { } extension ApplyInst { - fileprivate func isSafeReadOnlyApply( - calleeAnalysis: CalleeAnalysis - ) -> Bool { + fileprivate func isSafeReadOnlyApply(calleeAnalysis: CalleeAnalysis) -> Bool { guard functionConvention.results.allSatisfy({ $0.convention == .unowned }) else { return false } @@ -1082,22 +946,17 @@ extension ApplyInst { } extension BeginAccessInst { - private func isCoveredByScope( - otherInst: Instruction, - domTree: DominatorTree - ) -> Bool { + private func isCoveredByScope(otherInst: Instruction, domTree: DominatorTree) -> Bool { return self.dominates(otherInst, domTree) && endAccessInstructions .allSatisfy { endAccessInst in otherInst.dominates(endAccessInst, domTree) } } - private func handlesAllEndAccesses( - loop: Loop - ) -> Bool { + private func handlesAllEndAccesses(loop: Loop) -> Bool { return !endAccessInstructions.isEmpty && !endAccessInstructions .contains { user in - !loop.basicBlocks.contains(user.parentBlock) + !loop.loopBlocks.contains(user.parentBlock) } } @@ -1132,10 +991,7 @@ extension BeginAccessInst { continue } - if !isCoveredByScope( - otherInst: fullApplyInst, - domTree: domTree - ) { + if !isCoveredByScope(otherInst: fullApplyInst, domTree: domTree) { return false } } @@ -1147,10 +1003,7 @@ extension BeginAccessInst { continue } - if !isCoveredByScope( - otherInst: sideEffect, - domTree: domTree - ) { + if !isCoveredByScope(otherInst: sideEffect, domTree: domTree) { return false } } @@ -1170,13 +1023,13 @@ extension AccessPath { .pointer(let inst as Instruction), .stack(let inst as Instruction), .storeBorrow(let inst as Instruction), .tail(let inst as Instruction): - if loop.basicBlocks.contains(inst.parentBlock) { + if loop.loopBlocks.contains(inst.parentBlock) { return false } case .global, .argument: break case .yield(let beginApplyResult): - if loop.basicBlocks.contains(beginApplyResult.parentBlock) { + if loop.loopBlocks.contains(beginApplyResult.parentBlock) { return false } case .unidentified: @@ -1203,10 +1056,7 @@ extension AccessPath { accessesPath = false } - return !accessesPath && sideEffect.mayReadOrWrite( - address: storeAddr, - aliasAnalysis - ) + return !accessesPath && sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) } && !analyzedInstructions.loads .contains { loadInst in loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift index 51eff5c8ce598..594514795672f 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift @@ -152,3 +152,18 @@ extension MutatingContext { bridgedPassContext.notifyDependencyOnBodyOf(otherFunction.bridged) } } + +extension Instruction { + var arraySemanticsCallKind: BridgedArrayCallKind { + return BridgedPassContext.getArraySemanticsCallKind(self.bridged) + } + + func canHoistArraySemanticsCall(to toInst: Instruction, _ context: FunctionPassContext) -> Bool { + return context.bridgedPassContext.canHoistArraySemanticsCall(self.bridged, toInst.bridged) + } + + func hoistArraySemanticsCall(before toInst: Instruction, _ context: some MutatingContext) { + context.bridgedPassContext.hoistArraySemanticsCall(self.bridged, toInst.bridged) // Internally updates dom tree. + context.notifyInstructionsChanged() + } +} diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift index d2bd929ecba1c..3d66618555de5 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift @@ -72,6 +72,13 @@ struct Cloner { if isCloned(value: value) { return getClonedValue(of: value) } + + if let beginAccess = value as? BeginAccessInst { + // Skip access instructions, which might be generated for UnsafePointer globals which point to other globals. + let clonedOperand = cloneRecursively(value: beginAccess.address) + bridged.recordFoldedValue(beginAccess.bridged, clonedOperand.bridged) + return clonedOperand + } guard let inst = value.definingInstruction else { fatalError("expected instruction to clone or already cloned value") @@ -97,12 +104,7 @@ struct Cloner { } guard !checkBase(addr) else { - guard let inst = addr as? Instruction else { - // TODO: Might have to additionally register like the instruction below. - return addr - } - - bridged.recordClonedInstruction(inst.bridged, inst.bridged) + bridged.recordFoldedValue(addr.bridged, addr.bridged) return addr } @@ -137,10 +139,6 @@ struct Cloner { ) else { return nil } - -// for op in projectAddr.operands { -// _ = cloneUseDefChain(addr: op.value, checkBase: checkBase) -// } let clone = clone(instruction: projectAddr) clone.setOperand(at: sourceOperand.index, to: projectedSource, context) diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift index 4156ca8018610..85bd2a2849d5c 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift @@ -652,7 +652,8 @@ extension LoadInst { let newStruct = builder.createStruct(type: self.type, elements: elements) self.replace(with: newStruct, context) - if let recursiveSplitLoad = elements[index].trySplit(alongPath: pathRemainder, context) { + // TODO: Sometimes `index < elements.count` might fail, should investigate and find real fix. See `embedded/stdlib-strings-datatables.swift` without this check. + if index < elements.count, let recursiveSplitLoad = elements[index].trySplit(alongPath: pathRemainder, context) { elements.remove(at: index) elements += recursiveSplitLoad } diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift b/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift index 546d5c7c30893..8899046e18c0c 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift @@ -392,6 +392,10 @@ public struct AccessPath : CustomStringConvertible, Hashable { public func isEqualOrContains(_ other: AccessPath) -> Bool { return getProjection(to: other) != nil } + + public func contains(_ other: AccessPath) -> Bool { + return !(getProjection(to: other)?.isEmpty ?? true) + } public var materializableProjectionPath: SmallProjectionPath? { if projectionPath.isMaterializable { diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index a13b71f52dcd8..7834f5776fa40 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -152,6 +152,7 @@ struct BridgedCloner { SWIFT_IMPORT_UNSAFE BridgedValue getClonedValue(BridgedValue v); bool isValueCloned(BridgedValue v) const; void recordClonedInstruction(BridgedInstruction origInst, BridgedInstruction clonedInst) const; + void recordFoldedValue(BridgedValue orig, BridgedValue mapped) const; BridgedInstruction clone(BridgedInstruction inst); }; diff --git a/lib/SILOptimizer/Utils/OptimizerBridging.cpp b/lib/SILOptimizer/Utils/OptimizerBridging.cpp index f82b21a7ad7b9..65e10c3f7f5da 100644 --- a/lib/SILOptimizer/Utils/OptimizerBridging.cpp +++ b/lib/SILOptimizer/Utils/OptimizerBridging.cpp @@ -569,6 +569,10 @@ void BridgedCloner::recordClonedInstruction(BridgedInstruction origInst, Bridged cloner->recordClonedInstruction(origInst.unbridged(), clonedInst.unbridged()); } +void BridgedCloner::recordFoldedValue(BridgedValue orig, BridgedValue mapped) const { + cloner->recordFoldedValue(orig.getSILValue(), mapped.getSILValue()); +} + BridgedInstruction BridgedCloner::clone(BridgedInstruction inst) { return {cloner->cloneInst(inst.unbridged())->asSILNode()}; } diff --git a/test/Interop/Cxx/function/default-arguments-multifile.swift b/test/Interop/Cxx/function/default-arguments-multifile.swift index 2dd3bc315d7b0..eb103118778a2 100644 --- a/test/Interop/Cxx/function/default-arguments-multifile.swift +++ b/test/Interop/Cxx/function/default-arguments-multifile.swift @@ -3,12 +3,12 @@ // RUN: mkdir -p %t/artifacts // Multiple usages in the same module. -// RUN: %target-build-swift %t/main.swift %t/b.swift %t/c.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -o %t/artifacts/out +// RUN: %target-build-swift -save-temps %t/main.swift %t/b.swift %t/c.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -o %t/artifacts/out // RUN: %empty-directory(%t/artifacts) // Multiple usages across different modules. -// RUN: %target-build-swift -emit-library -module-name BarLibrary -emit-module -emit-module-path %t/artifacts/BarLibrary.swiftmodule %t/b.swift %t/c.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -o %t/artifacts/%target-library-name(BarLibrary) -// RUN: %target-build-swift %t/uses-library.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -I %t/artifacts -L %t/artifacts -lBarLibrary -o %t/artifacts/uses-library +// RUN: %target-build-swift -save-temps -emit-library -module-name BarLibrary -emit-module -emit-module-path %t/artifacts/BarLibrary.swiftmodule %t/b.swift %t/c.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -o %t/artifacts/%target-library-name(BarLibrary) +// RUN: %target-build-swift -save-temps %t/uses-library.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -I %t/artifacts -L %t/artifacts -lBarLibrary -o %t/artifacts/uses-library // FIXME: Windows test can be enabled once merge-modules step is removed from the old driver, // or the Windows CI starts to use the new driver to compile the compiler. From 8f6edc95b5573e88eefdd1eacce036644cb7b610 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Tue, 5 Aug 2025 17:07:53 +0100 Subject: [PATCH 27/44] Add more efficient lookup of blocks inside a loop. Use block markers when iterating through side effects. --- .../Sources/Optimizer/Analysis/LoopTree.swift | 105 +++++++++++------- .../LoopInvariantCodeMotion.swift | 69 ++++++------ .../Sources/SIL/DataStructures/Stack.swift | 6 +- include/swift/SIL/SILBridging.h | 2 + .../SILOptimizer/OptimizerBridgingImpl.h | 4 + 5 files changed, 108 insertions(+), 78 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index b96bfdcdfab34..d5006fee1ad3f 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -15,7 +15,7 @@ import OptimizerBridging /// Describes top level loops. struct LoopTree { - private let bridged: BridgedLoopTree + fileprivate let bridged: BridgedLoopTree let loops: TopLevelLoopArray @@ -23,18 +23,6 @@ struct LoopTree { self.bridged = bridged self.loops = TopLevelLoopArray(bridged) } - - func splitCriticalEdge(basicBlock: BasicBlock, edgeIndex: Int, domTree: DominatorTree) -> BasicBlock? { - guard basicBlock.isCriticalEdge(edgeIndex: edgeIndex) else { - return nil - } - - return splitEdge(basicBlock: basicBlock, edgeIndex: edgeIndex, domTree: domTree) - } - - func splitEdge(basicBlock: BasicBlock, edgeIndex: Int, domTree: DominatorTree) -> BasicBlock? { - return bridged.splitEdge(basicBlock.bridged, edgeIndex, domTree.bridged).block - } } /// Describes a loop with its children. @@ -44,34 +32,38 @@ struct Loop { let innerLoops: LoopArray let loopBlocks: LoopBlocks - var preheader: BasicBlock? { - bridged.getPreheader().block - } + private(set) var exitingAndLatchBlocks: [BasicBlock] = [] + private(set) var exitBlocks: [BasicBlock] = [] + private(set) var exitingBlocks: [BasicBlock] = [] - var header: BasicBlock { - bridged.getHeader().block - } - - var exitingAndLatchBlocks: some Sequence { - return header.predecessors.lazy - .filter { predecessor in - loopBlocks.contains(predecessor) && !isLoopExiting(loopBlock: predecessor) - } + exitingBlocks - } - - var exitBlocks: some Sequence { - return loopBlocks.lazy + init(bridged: BridgedLoop) { + self.bridged = bridged + self.innerLoops = LoopArray(bridged) + self.loopBlocks = LoopBlocks(bridged) + + self.exitingBlocks = loopBlocks + .filter { bb in + isLoopExiting(loopBlock: bb) + } + + self.exitingAndLatchBlocks = header.predecessors + .filter { predecessor in + contains(block: predecessor) && !isLoopExiting(loopBlock: predecessor) + } + exitingBlocks + + self.exitBlocks = loopBlocks .flatMap(\.successors) .filter { succesor in - !loopBlocks.contains(succesor) + !contains(block: succesor) } } - var exitingBlocks: some Sequence { - return loopBlocks.lazy - .filter { bb in - isLoopExiting(loopBlock: bb) - } + var preheader: BasicBlock? { + bridged.getPreheader().block + } + + var header: BasicBlock { + bridged.getHeader().block } var isSingleExit: Bool { @@ -83,12 +75,16 @@ struct Loop { } private func isLoopExiting(loopBlock: BasicBlock) -> Bool { - return loopBlock.successors.contains { !loopBlocks.contains($0) } + return loopBlock.successors.contains { !contains(block: $0) } } func getBlocksThatDominateAllExitingAndLatchBlocks(domTree: DominatorTree) -> some Sequence { return getBlocksThatDominateAllExitingAndLatchBlocksHelper(bb: header, domTree: domTree) } + + func contains(block: BasicBlock) -> Bool { + return bridged.contains(block.bridged) + } private func getBlocksThatDominateAllExitingAndLatchBlocksHelper( bb: BasicBlock, @@ -108,12 +104,6 @@ struct Loop { ) } } - - init(bridged: BridgedLoop) { - self.bridged = bridged - self.innerLoops = LoopArray(bridged) - self.loopBlocks = LoopBlocks(bridged) - } } struct TopLevelLoopArray: BridgedRandomAccessCollection { @@ -172,3 +162,34 @@ struct LoopBlocks: BridgedRandomAccessCollection { return bridgedLoop.getBasicBlock(index).block } } + +func splitEdge( + from block: BasicBlock, + toEdgeIndex: Int, + dominatorTree: DominatorTree, + loopTree: LoopTree, + _ context: some MutatingContext +) -> BasicBlock? { + guard let result = loopTree.bridged.splitEdge(block.bridged, toEdgeIndex, dominatorTree.bridged).block else { + return nil + } + + context.notifyBranchesChanged() + return result +} + +/// If the specified edge is critical, the function returns inserted block. Otherwise returns `nil`. +@discardableResult +func splitCriticalEdge( + from block: BasicBlock, + toEdgeIndex: Int, + dominatorTree: DominatorTree, + loopTree: LoopTree, + _ context: some MutatingContext +) -> BasicBlock? { + guard block.isCriticalEdge(edgeIndex: toEdgeIndex) else { + return nil + } + + return splitEdge(from: block, toEdgeIndex: toEdgeIndex, dominatorTree: dominatorTree, loopTree: loopTree, context) +} diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 5799c24d5b7f6..ace16f25a7712 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -157,12 +157,12 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext defer { analyzedInstructions.deinitialize() } for bb in loop.loopBlocks { - var blockSideEffects = Stack(context) - defer { blockSideEffects.deinitialize() } + let blockSideEffectBottomMarker = analyzedInstructions.loopSideEffects.top for inst in bb.instructions { + // TODO: Remove once support for values with ownership implemented. if inst.hasOwnershipOperandsOrResults { - analyzedInstructions.analyzeSideEffects(ofInst: inst, blockSideEffects: &blockSideEffects) + analyzedInstructions.analyzeSideEffects(ofInst: inst) if let fullApply = inst as? FullApplySite { analyzedInstructions.append(fullApply) @@ -186,15 +186,15 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext } analyzedInstructions.append(storeInst) movableInstructions.loadsAndStores.append(storeInst) - analyzedInstructions.analyzeSideEffects(ofInst: storeInst, blockSideEffects: &blockSideEffects) + analyzedInstructions.analyzeSideEffects(ofInst: storeInst) case let beginAccessInst as BeginAccessInst: analyzedInstructions.append(beginAccessInst) - analyzedInstructions.analyzeSideEffects(ofInst: beginAccessInst, blockSideEffects: &blockSideEffects) + analyzedInstructions.analyzeSideEffects(ofInst: beginAccessInst) case let refElementAddrInst as RefElementAddrInst: movableInstructions.simplyHoistableInsts.append(refElementAddrInst) case let condFailInst as CondFailInst: movableInstructions.hoistUp.append(condFailInst) - analyzedInstructions.analyzeSideEffects(ofInst: condFailInst, blockSideEffects: &blockSideEffects) + analyzedInstructions.analyzeSideEffects(ofInst: condFailInst) case let applyInst as ApplyInst: if applyInst.isSafeReadOnlyApply( calleeAnalysis: context.calleeAnalysis @@ -203,7 +203,8 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext } else if let callee = applyInst.referencedFunction, callee.isGlobalInitFunction, !applyInst.globalInitMayConflictWith( - sideEffects: blockSideEffects, + loopSideEffects: analyzedInstructions.loopSideEffects, + blockSideEffectBottomMarker: blockSideEffectBottomMarker, aliasAnalysis: context.aliasAnalysis ) { analyzedInstructions.appendGlobalInitCall(applyInst) @@ -218,7 +219,8 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext switch builtinInst.id { case .Once, .OnceWithContext: if !builtinInst.globalInitMayConflictWith( - sideEffects: blockSideEffects, + loopSideEffects: analyzedInstructions.loopSideEffects, + blockSideEffectBottomMarker: blockSideEffectBottomMarker, aliasAnalysis: context.aliasAnalysis ) { analyzedInstructions.appendGlobalInitCall(builtinInst) @@ -228,7 +230,7 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext default: break } - analyzedInstructions.analyzeSideEffects(ofInst: inst, blockSideEffects: &blockSideEffects) + analyzedInstructions.analyzeSideEffects(ofInst: inst) if inst.canBeHoisted(outOf: loop, context) { movableInstructions.hoistUp.append(inst) @@ -262,7 +264,7 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext for globalInitCall in analyzedInstructions.globalInitCalls { if !globalInitCall.globalInitMayConflictWith( preheader: preheader, - sideEffects: analyzedInstructions.loopSideEffects, + loopSideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis, postDomTree: context.postDominatorTree ) { @@ -338,13 +340,9 @@ private func optimizeLoop( } extension AnalyzedInstructions { - mutating func analyzeSideEffects( - ofInst inst: Instruction, - blockSideEffects: inout Stack - ) { + mutating func analyzeSideEffects(ofInst inst: Instruction) { if inst.mayHaveSideEffects { appendSideEffect(inst) - blockSideEffects.append(inst) } else if inst.mayReadFromMemory { hasOtherMemReadingInsts = true } @@ -465,11 +463,13 @@ extension MovableInstructions { } for exitingOrLatchBlock in exitingAndLatchBlocks { - for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !loop.loopBlocks.contains(succesor) { - _ = context.loopTree.splitCriticalEdge( - basicBlock: exitingOrLatchBlock.terminator.parentBlock, - edgeIndex: index, - domTree: context.dominatorTree + for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !loop.contains(block: succesor) { + splitCriticalEdge( + from: exitingOrLatchBlock.terminator.parentBlock, + toEdgeIndex: index, + dominatorTree: context.dominatorTree, + loopTree: context.loopTree, + context ) } } @@ -564,7 +564,7 @@ extension MovableInstructions { } for exitingOrLatchBlock in exitingAndLatchBlocks { - for succesor in exitingOrLatchBlock.successors where !loop.loopBlocks.contains(succesor) { + for succesor in exitingOrLatchBlock.successors where !loop.contains(block: succesor) { guard let firstInst = succesor.instructions.first else { continue } @@ -696,7 +696,7 @@ extension Instruction { @discardableResult fileprivate func hoist(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { guard let preheader = loop.preheader, - operands.allSatisfy({ !loop.loopBlocks.contains($0.value.parentBlock) }) else { + operands.allSatisfy({ !loop.contains(block: $0.value.parentBlock) }) else { return false } @@ -734,10 +734,12 @@ extension Instruction { for (succesorIndex, succesor) in exitingBlock.successors.enumerated().reversed() where !newExitBlocks.contains(succesor) && exitBlocks.contains(succesor) { - let outsideBlock = context.loopTree.splitCriticalEdge( - basicBlock: exitingBlock, - edgeIndex: succesorIndex, - domTree: context.dominatorTree + let outsideBlock = splitCriticalEdge( + from: exitingBlock, + toEdgeIndex: succesorIndex, + dominatorTree: context.dominatorTree, + loopTree: context.loopTree, + context ) ?? succesor newExitBlocks.push(outsideBlock) @@ -780,10 +782,11 @@ extension Instruction { } fileprivate func globalInitMayConflictWith( - sideEffects: Stack, + loopSideEffects: Stack, + blockSideEffectBottomMarker: Stack.Marker, aliasAnalysis: AliasAnalysis ) -> Bool { - return sideEffects + return Stack.Segment(in: loopSideEffects, low: blockSideEffectBottomMarker, high: loopSideEffects.top) .contains { sideEffect in globalInitMayConflictWith( sideEffect: sideEffect, @@ -794,7 +797,7 @@ extension Instruction { fileprivate func globalInitMayConflictWith( preheader: BasicBlock, - sideEffects: Stack, + loopSideEffects: Stack, aliasAnalysis: AliasAnalysis, postDomTree: PostDominatorTree ) -> Bool { @@ -802,7 +805,7 @@ extension Instruction { return true } - return sideEffects + return loopSideEffects .contains { sideEffect in parentBlock.strictlyPostDominates( sideEffect.parentBlock, @@ -956,7 +959,7 @@ extension BeginAccessInst { private func handlesAllEndAccesses(loop: Loop) -> Bool { return !endAccessInstructions.isEmpty && !endAccessInstructions .contains { user in - !loop.loopBlocks.contains(user.parentBlock) + !loop.contains(block: user.parentBlock) } } @@ -1023,13 +1026,13 @@ extension AccessPath { .pointer(let inst as Instruction), .stack(let inst as Instruction), .storeBorrow(let inst as Instruction), .tail(let inst as Instruction): - if loop.loopBlocks.contains(inst.parentBlock) { + if loop.contains(block: inst.parentBlock) { return false } case .global, .argument: break case .yield(let beginApplyResult): - if loop.loopBlocks.contains(beginApplyResult.parentBlock) { + if loop.contains(block: beginApplyResult.parentBlock) { return false } case .unidentified: diff --git a/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift b/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift index 026e42fac92ab..402d5b887e9a6 100644 --- a/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift +++ b/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift @@ -140,7 +140,7 @@ public struct Stack : CollectionLikeSequence { public mutating func deinitialize() { removeAll() } } -extension Stack { +public extension Stack { /// Mark a stack location for future iteration. /// /// TODO: Marker should be ~Escapable. @@ -155,7 +155,7 @@ extension Stack { let low: Marker let high: Marker - init(in stack: Stack, low: Marker, high: Marker) { + public init(in stack: Stack, low: Marker, high: Marker) { if low.slab.data == nil { assert(low.index == 0, "invalid empty stack marker") // `low == nil` and `high == nil` is a valid empty segment, @@ -173,7 +173,7 @@ extension Stack { self.high = high } - func makeIterator() -> Stack.Iterator { + public func makeIterator() -> Stack.Iterator { return Iterator(slab: low.slab, index: low.index, lastSlab: high.slab, endIndex: high.index) } diff --git a/include/swift/SIL/SILBridging.h b/include/swift/SIL/SILBridging.h index 38d264a465831..c0e36a05830ba 100644 --- a/include/swift/SIL/SILBridging.h +++ b/include/swift/SIL/SILBridging.h @@ -92,6 +92,8 @@ struct BridgedLoop { BRIDGED_INLINE OptionalBridgedBasicBlock getPreheader() const; BRIDGED_INLINE BridgedBasicBlock getHeader() const; + + BRIDGED_INLINE bool contains(BridgedBasicBlock block) const; }; enum class BridgedArrayCallKind { diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index 5b8773cc8944d..a703ee00a56b9 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -137,6 +137,10 @@ BridgedBasicBlock BridgedLoop::getHeader() const { return {l->getHeader()}; } +bool BridgedLoop::contains(BridgedBasicBlock block) const { + return l->contains(block.unbridged()); +} + //===----------------------------------------------------------------------===// // BridgedPassContext //===----------------------------------------------------------------------===// From 5cee03eaa4336185d695e7a9b01f315581668d95 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Tue, 5 Aug 2025 22:59:53 +0100 Subject: [PATCH 28/44] Add swift:: namespace before ArraySemanticsCall. --- include/swift/SILOptimizer/OptimizerBridgingImpl.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index a703ee00a56b9..cb7a5b1458cdb 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -208,17 +208,17 @@ BridgedDeclObj BridgedPassContext::getSwiftMutableSpanDecl() const { // Array semantics call BridgedArrayCallKind BridgedPassContext::getArraySemanticsCallKind(BridgedInstruction inst) { - ArraySemanticsCall semCall(inst.unbridged()); + swift::ArraySemanticsCall semCall(inst.unbridged()); return static_cast(semCall.getKind()); } bool BridgedPassContext::canHoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction toInst) const { - ArraySemanticsCall semCall(inst.unbridged()); + swift::ArraySemanticsCall semCall(inst.unbridged()); return semCall.canHoist(toInst.unbridged(), getDomTree().di); } void BridgedPassContext::hoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction beforeInst) const { - ArraySemanticsCall semCall(inst.unbridged()); + swift::ArraySemanticsCall semCall(inst.unbridged()); semCall.hoist(beforeInst.unbridged(), getDomTree().di); } From 1519482ff7ed8af9411f8f4f1770bfb6f133df9f Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Wed, 6 Aug 2025 16:13:22 +0100 Subject: [PATCH 29/44] Simplify load splitting during loop analysis. Revert loop block data structures to computed properties. --- .../Sources/Optimizer/Analysis/LoopTree.swift | 40 ++++--- .../LoopInvariantCodeMotion.swift | 111 +++++++++--------- 2 files changed, 74 insertions(+), 77 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index d5006fee1ad3f..dfc4c778f68fa 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -32,32 +32,34 @@ struct Loop { let innerLoops: LoopArray let loopBlocks: LoopBlocks - private(set) var exitingAndLatchBlocks: [BasicBlock] = [] - private(set) var exitBlocks: [BasicBlock] = [] - private(set) var exitingBlocks: [BasicBlock] = [] + var exitingAndLatchBlocks: [BasicBlock] { + return header.predecessors + .filter { predecessor in + contains(block: predecessor) && !isLoopExiting(loopBlock: predecessor) + } + exitingBlocks + } - init(bridged: BridgedLoop) { - self.bridged = bridged - self.innerLoops = LoopArray(bridged) - self.loopBlocks = LoopBlocks(bridged) - - self.exitingBlocks = loopBlocks - .filter { bb in - isLoopExiting(loopBlock: bb) - } - - self.exitingAndLatchBlocks = header.predecessors - .filter { predecessor in - contains(block: predecessor) && !isLoopExiting(loopBlock: predecessor) - } + exitingBlocks - - self.exitBlocks = loopBlocks + var exitBlocks: [BasicBlock] { + return loopBlocks .flatMap(\.successors) .filter { succesor in !contains(block: succesor) } } + var exitingBlocks: [BasicBlock] { + return loopBlocks + .filter { bb in + isLoopExiting(loopBlock: bb) + } + } + + init(bridged: BridgedLoop) { + self.bridged = bridged + self.innerLoops = LoopArray(bridged) + self.loopBlocks = LoopBlocks(bridged) + } + var preheader: BasicBlock? { bridged.getPreheader().block } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index ace16f25a7712..d11f1e990003e 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -20,8 +20,10 @@ let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion } private func optimizeTopLevelLoop(topLevelLoop: Loop, _ context: FunctionPassContext) { - var innerLoops = getWorkList(topLevelLoop: topLevelLoop, context) + var innerLoops = Stack(context) defer { innerLoops.deinitialize() } + + getWorkList(forLoop: topLevelLoop, result: &innerLoops) while let thisLoop = innerLoops.pop() { var thisLoopChanged = false @@ -34,29 +36,12 @@ private func optimizeTopLevelLoop(topLevelLoop: Loop, _ context: FunctionPassCon } } -private func getWorkList(topLevelLoop: Loop, _ context: Context) -> Stack { - var tmp1 = Stack(context) - var tmp2 = Stack(context) - var workList = Stack(context) - defer { - tmp1.deinitialize() - tmp2.deinitialize() - } - - tmp1.push(topLevelLoop) - - while !tmp1.isEmpty || !tmp2.isEmpty { - while let loop = tmp2.pop() { - workList.push(loop) - } - - while let loop = tmp1.pop() { - tmp2.push(loop) - tmp1.append(contentsOf: loop.innerLoops) - } +private func getWorkList(forLoop loop: Loop, result: inout Stack) { + result.push(loop) + + for innerLoop in loop.innerLoops { + getWorkList(forLoop: innerLoop, result: &result) } - - return workList } struct MovableInstructions { @@ -76,7 +61,6 @@ struct AnalyzedInstructions { private(set) var readOnlyApplies: Stack private(set) var loads: Stack private(set) var stores: Stack - private(set) var fixLifetimes: Stack private(set) var beginAccesses: Stack private(set) var fullApplies: Stack @@ -86,6 +70,8 @@ struct AnalyzedInstructions { private(set) var hasOtherMemReadingInsts = false + private(set) lazy var sideEffectsMayRelease = loopSideEffects.contains(where: { $0.mayRelease }) + init (_ context: FunctionPassContext) { self.globalInitCalls = Stack(context) self.loopSideEffects = Stack(context) @@ -93,7 +79,6 @@ struct AnalyzedInstructions { self.readOnlyApplies = Stack(context) self.loads = Stack(context) self.stores = Stack(context) - self.fixLifetimes = Stack(context) self.beginAccesses = Stack(context) self.fullApplies = Stack(context) } @@ -131,7 +116,6 @@ struct AnalyzedInstructions { } mutating func append(_ storeInst: StoreInst) { stores.append(storeInst) } - mutating func append(_ fixLifetime: FixLifetimeInst) { fixLifetimes.append(fixLifetime) } mutating func append(_ beginAccess: BeginAccessInst) { beginAccesses.append(beginAccess) } mutating func append(_ fullApply: FullApplySite) { fullApplies.append(fullApply) } @@ -141,7 +125,6 @@ struct AnalyzedInstructions { loopSideEffects.deinitialize() loads.deinitialize() stores.deinitialize() - fixLifetimes.deinitialize() beginAccesses.deinitialize() fullApplies.deinitialize() } @@ -172,11 +155,8 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext } switch inst { - case let fixLifetimeInst as FixLifetimeInst where fixLifetimeInst.parentBlock.dominates(preheader, context.dominatorTree): - analyzedInstructions.append(fixLifetimeInst) case let loadInst as LoadInst: analyzedInstructions.append(loadInst) - movableInstructions.loadsAndStores.append(loadInst) case let storeInst as StoreInst: switch storeInst.storeOwnership { case .assign, .initialize: @@ -185,7 +165,6 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext break } analyzedInstructions.append(storeInst) - movableInstructions.loadsAndStores.append(storeInst) analyzedInstructions.analyzeSideEffects(ofInst: storeInst) case let beginAccessInst as BeginAccessInst: analyzedInstructions.append(beginAccessInst) @@ -283,7 +262,6 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext ), !movableInstructions.loadAndStoreAddrs.contains(accessPath), analyzedInstructions.splitLoads( - movableInstructions: &movableInstructions, storeAddr: storeInst.destination, accessPath: accessPath, context @@ -292,33 +270,51 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext } } } - - if !analyzedInstructions.fixLifetimes.isEmpty { - let sideEffectsMayRelease = analyzedInstructions.loopSideEffects.contains(where: { $0.mayRelease }) - - for fixLifetime in analyzedInstructions.fixLifetimes { - guard fixLifetime.operand.value.type.isAddress else { continue } - - if sideEffectsMayRelease || !fixLifetime.mayWriteTo( - sideEffects: analyzedInstructions.loopSideEffects, - aliasAnalysis: context.aliasAnalysis - ) { - movableInstructions.sinkDown.append(fixLifetime) + + for bb in loop.loopBlocks { + for inst in bb.instructions { + // TODO: Remove once support for values with ownership implemented. + if inst.hasOwnershipOperandsOrResults { + continue + } + + switch inst { + case let fixLifetimeInst as FixLifetimeInst: + guard fixLifetimeInst.parentBlock.dominates(preheader, context.dominatorTree) else { + continue + } + + if analyzedInstructions.sideEffectsMayRelease || !fixLifetimeInst.mayWriteTo( + sideEffects: analyzedInstructions.loopSideEffects, + aliasAnalysis: context.aliasAnalysis + ) { + movableInstructions.sinkDown.append(fixLifetimeInst) + } + case let loadInst as LoadInst: + movableInstructions.loadsAndStores.append(loadInst) + case let storeInst as StoreInst: + switch storeInst.storeOwnership { + case .assign, .initialize: + continue // TODO: Add support + case .unqualified, .trivial: + break + } + movableInstructions.loadsAndStores.append(storeInst) + case let beginAccessInst as BeginAccessInst: + if beginAccessInst.canBeHoisted( + outOf: loop, + analyzedInstructions: analyzedInstructions, + aliasAnalysis: context.aliasAnalysis, + domTree: context.dominatorTree + ) { + movableInstructions.scopedInsts.append(beginAccessInst) + } + default: + break } } } - - for beginAccessInst in analyzedInstructions.beginAccesses { - if beginAccessInst.canBeHoisted( - outOf: loop, - analyzedInstructions: analyzedInstructions, - aliasAnalysis: context.aliasAnalysis, - domTree: context.dominatorTree - ) { - movableInstructions.scopedInsts.append(beginAccessInst) - } - } - + return movableInstructions } @@ -349,7 +345,6 @@ extension AnalyzedInstructions { } mutating func splitLoads( - movableInstructions: inout MovableInstructions, storeAddr: Value, accessPath: AccessPath, _ context: FunctionPassContext @@ -374,7 +369,6 @@ extension AnalyzedInstructions { if let splitLoads = loadInst.trySplit(alongPath: accessPath.projectionPath, context) { splitCounter += splitLoads.count - movableInstructions.loadsAndStores.replace([loadInst], with: splitLoads) newLoads.append(contentsOf: splitLoads) } } @@ -708,6 +702,7 @@ extension Instruction { } if let singleValueInst = self as? SingleValueInstruction, + !(self is BeginAccessInst), let identicalInst = (preheader.instructions.first { otherInst in return singleValueInst != otherInst && singleValueInst.isIdenticalTo(otherInst) }) { From 6f2707b851e73472d2a925bf3b0d9294ccba16fa Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Fri, 8 Aug 2025 10:59:47 +0100 Subject: [PATCH 30/44] Add instruction merging in loop preheader during analysis. --- .../LoopInvariantCodeMotion.swift | 17 +++++++++++++++++ test/SILOptimizer/licm.sil | 1 - 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index d11f1e990003e..007d8e57de438 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -130,10 +130,27 @@ struct AnalyzedInstructions { } } +// Merges address instructions in loop preheaders. Quadratic complexity. Remove before merging. +func mergeSomeInstructionsInPreheader(preheader: BasicBlock, _ context: FunctionPassContext) { + for (outerInstIndex, outerInst) in preheader.instructions.enumerated() where !outerInst.isDeleted { + for (innerInstIndex, innerInst) in preheader.instructions.enumerated() where innerInstIndex > outerInstIndex && !innerInst.isDeleted && outerInst.isIdenticalTo(innerInst) { + guard let outerSingleValueInst = outerInst as? SingleValueInstruction, + let innerSingleValueInst = innerInst as? SingleValueInstruction, + (outerSingleValueInst is RefElementAddrInst || outerSingleValueInst is StructElementAddrInst || outerSingleValueInst is TupleElementAddrInst || outerSingleValueInst is PointerToAddressInst) else { + continue + } + + innerSingleValueInst.replace(with: outerSingleValueInst, context) + } + } +} + private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext) -> MovableInstructions? { guard let preheader = loop.preheader else { return nil } + + mergeSomeInstructionsInPreheader(preheader: preheader, context) var movableInstructions = MovableInstructions() var analyzedInstructions = AnalyzedInstructions(context) diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index 404aea3119a38..a72831cfe29c9 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -1043,7 +1043,6 @@ struct State { // CHECK: bb0(%0 : $Builtin.RawPointer): // CHECK: [[HOISTADR:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 0 // ...Preload stored element #1 -// CHECK: tuple_element_addr // CHECK: [[PRELOADADR:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 1 // CHECK: [[PRELOAD:%.*]] = load [[PRELOADADR]] : $*Int64 // ...Split element 0 From 688881c72580e0d6aab9b454d0ede2e25dc52156 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Fri, 8 Aug 2025 15:24:28 +0100 Subject: [PATCH 31/44] Disable operation merging when hoisting. More refactoring. --- .../Sources/Optimizer/Analysis/LoopTree.swift | 8 +- .../LoopInvariantCodeMotion.swift | 155 ++++++------------ .../Sources/SIL/DataStructures/Stack.swift | 84 ++++++++++ test/SILOptimizer/licm.sil | 1 + 4 files changed, 135 insertions(+), 113 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index dfc4c778f68fa..5ff516c3051df 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -42,16 +42,12 @@ struct Loop { var exitBlocks: [BasicBlock] { return loopBlocks .flatMap(\.successors) - .filter { succesor in - !contains(block: succesor) - } + .filter { !contains(block: $0) } } var exitingBlocks: [BasicBlock] { return loopBlocks - .filter { bb in - isLoopExiting(loopBlock: bb) - } + .filter { isLoopExiting(loopBlock: $0) } } init(bridged: BridgedLoop) { diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 007d8e57de438..a1771c2f8ebde 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -55,70 +55,30 @@ struct MovableInstructions { } struct AnalyzedInstructions { - private(set) var globalInitCalls: Stack - private(set) var loopSideEffects: Stack + var globalInitCalls: Stack + var loopSideEffects: StackWithCount - private(set) var readOnlyApplies: Stack - private(set) var loads: Stack - private(set) var stores: Stack - private(set) var beginAccesses: Stack - private(set) var fullApplies: Stack + var readOnlyApplies: StackWithCount + var loads: StackWithCount + var stores: Stack + var beginAccesses: Stack + var fullApplies: Stack - private(set) var readOnlyAppliesCount = 0 - private(set) var loopSideEffectCount = 0 - private(set) var loadsCount = 0 + var hasOtherMemReadingInsts = false - private(set) var hasOtherMemReadingInsts = false - - private(set) lazy var sideEffectsMayRelease = loopSideEffects.contains(where: { $0.mayRelease }) + lazy var sideEffectsMayRelease = loopSideEffects.contains(where: { $0.mayRelease }) init (_ context: FunctionPassContext) { self.globalInitCalls = Stack(context) - self.loopSideEffects = Stack(context) + self.loopSideEffects = StackWithCount(context) - self.readOnlyApplies = Stack(context) - self.loads = Stack(context) + self.readOnlyApplies = StackWithCount(context) + self.loads = StackWithCount(context) self.stores = Stack(context) self.beginAccesses = Stack(context) self.fullApplies = Stack(context) } - mutating func appendGlobalInitCall(_ inst: Instruction) { globalInitCalls.append(inst) } - - mutating func appendSideEffect(_ inst: Instruction) { - loopSideEffects.append(inst) - loopSideEffectCount += 1 - } - - mutating func append(_ applyInst: ApplyInst) { - readOnlyApplies.append(applyInst) - readOnlyAppliesCount += 1 - } - - mutating func append(_ loadInst: LoadInst) { - loads.append(loadInst) - loadsCount += 1 - } - - mutating func popLoad() -> LoadInst? { - guard !loads.isEmpty else { - return nil - } - - loadsCount -= 1 - return loads.pop() - } - - mutating func append(newLoads: Stack) { - for load in newLoads { - append(load) - } - } - - mutating func append(_ storeInst: StoreInst) { stores.append(storeInst) } - mutating func append(_ beginAccess: BeginAccessInst) { beginAccesses.append(beginAccess) } - mutating func append(_ fullApply: FullApplySite) { fullApplies.append(fullApply) } - mutating func deinitialize() { readOnlyApplies.deinitialize() globalInitCalls.deinitialize() @@ -130,27 +90,10 @@ struct AnalyzedInstructions { } } -// Merges address instructions in loop preheaders. Quadratic complexity. Remove before merging. -func mergeSomeInstructionsInPreheader(preheader: BasicBlock, _ context: FunctionPassContext) { - for (outerInstIndex, outerInst) in preheader.instructions.enumerated() where !outerInst.isDeleted { - for (innerInstIndex, innerInst) in preheader.instructions.enumerated() where innerInstIndex > outerInstIndex && !innerInst.isDeleted && outerInst.isIdenticalTo(innerInst) { - guard let outerSingleValueInst = outerInst as? SingleValueInstruction, - let innerSingleValueInst = innerInst as? SingleValueInstruction, - (outerSingleValueInst is RefElementAddrInst || outerSingleValueInst is StructElementAddrInst || outerSingleValueInst is TupleElementAddrInst || outerSingleValueInst is PointerToAddressInst) else { - continue - } - - innerSingleValueInst.replace(with: outerSingleValueInst, context) - } - } -} - private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext) -> MovableInstructions? { guard let preheader = loop.preheader else { return nil } - - mergeSomeInstructionsInPreheader(preheader: preheader, context) var movableInstructions = MovableInstructions() var analyzedInstructions = AnalyzedInstructions(context) @@ -165,7 +108,7 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext analyzedInstructions.analyzeSideEffects(ofInst: inst) if let fullApply = inst as? FullApplySite { - analyzedInstructions.append(fullApply) + analyzedInstructions.fullApplies.append(fullApply) } continue @@ -173,7 +116,7 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext switch inst { case let loadInst as LoadInst: - analyzedInstructions.append(loadInst) + analyzedInstructions.loads.append(loadInst) case let storeInst as StoreInst: switch storeInst.storeOwnership { case .assign, .initialize: @@ -181,21 +124,20 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext case .unqualified, .trivial: break } - analyzedInstructions.append(storeInst) + analyzedInstructions.stores.append(storeInst) analyzedInstructions.analyzeSideEffects(ofInst: storeInst) case let beginAccessInst as BeginAccessInst: - analyzedInstructions.append(beginAccessInst) + analyzedInstructions.beginAccesses.append(beginAccessInst) analyzedInstructions.analyzeSideEffects(ofInst: beginAccessInst) case let refElementAddrInst as RefElementAddrInst: movableInstructions.simplyHoistableInsts.append(refElementAddrInst) case let condFailInst as CondFailInst: - movableInstructions.hoistUp.append(condFailInst) analyzedInstructions.analyzeSideEffects(ofInst: condFailInst) case let applyInst as ApplyInst: if applyInst.isSafeReadOnlyApply( calleeAnalysis: context.calleeAnalysis ) { - analyzedInstructions.append(applyInst) + analyzedInstructions.readOnlyApplies.append(applyInst) } else if let callee = applyInst.referencedFunction, callee.isGlobalInitFunction, !applyInst.globalInitMayConflictWith( @@ -203,14 +145,14 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext blockSideEffectBottomMarker: blockSideEffectBottomMarker, aliasAnalysis: context.aliasAnalysis ) { - analyzedInstructions.appendGlobalInitCall(applyInst) + analyzedInstructions.globalInitCalls.append(applyInst) } fallthrough default: switch inst { case let fullApply as FullApplySite: - analyzedInstructions.append(fullApply) + analyzedInstructions.fullApplies.append(fullApply) case let builtinInst as BuiltinInst: switch builtinInst.id { case .Once, .OnceWithContext: @@ -219,7 +161,7 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext blockSideEffectBottomMarker: blockSideEffectBottomMarker, aliasAnalysis: context.aliasAnalysis ) { - analyzedInstructions.appendGlobalInitCall(builtinInst) + analyzedInstructions.globalInitCalls.append(builtinInst) } default: break } @@ -236,7 +178,7 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext } // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. - if analyzedInstructions.readOnlyAppliesCount * analyzedInstructions.loopSideEffectCount < 8000 { + if analyzedInstructions.readOnlyApplies.count * analyzedInstructions.loopSideEffects.count < 8000 { for readOnlyApply in analyzedInstructions.readOnlyApplies { if !readOnlyApply.mayWriteTo( sideEffects: analyzedInstructions.loopSideEffects, @@ -248,15 +190,6 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext } } - // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. - if analyzedInstructions.loadsCount * analyzedInstructions.loopSideEffectCount < 8000 { - for load in analyzedInstructions.loads { - if !load.mayWriteTo(sideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis) { - movableInstructions.hoistUp.append(load) - } - } - } - for globalInitCall in analyzedInstructions.globalInitCalls { if !globalInitCall.globalInitMayConflictWith( preheader: preheader, @@ -308,6 +241,12 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext movableInstructions.sinkDown.append(fixLifetimeInst) } case let loadInst as LoadInst: + // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. + if analyzedInstructions.loads.count * analyzedInstructions.loopSideEffects.count < 8000, + !loadInst.mayWriteTo(sideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis) { + movableInstructions.hoistUp.append(loadInst) + } + movableInstructions.loadsAndStores.append(loadInst) case let storeInst as StoreInst: switch storeInst.storeOwnership { @@ -317,6 +256,8 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext break } movableInstructions.loadsAndStores.append(storeInst) + case let condFailInst as CondFailInst: + movableInstructions.hoistUp.append(condFailInst) case let beginAccessInst as BeginAccessInst: if beginAccessInst.canBeHoisted( outOf: loop, @@ -355,7 +296,7 @@ private func optimizeLoop( extension AnalyzedInstructions { mutating func analyzeSideEffects(ofInst inst: Instruction) { if inst.mayHaveSideEffects { - appendSideEffect(inst) + loopSideEffects.append(inst) } else if inst.mayReadFromMemory { hasOtherMemReadingInsts = true } @@ -368,12 +309,12 @@ extension AnalyzedInstructions { ) -> Bool { var newLoads = Stack(context) defer { - append(newLoads: newLoads) + loads.append(contentsOf: newLoads) newLoads.deinitialize() } var splitCounter = 0 - while let loadInst = popLoad() { + while let loadInst = loads.pop() { guard splitCounter <= 6 else { newLoads.push(loadInst) return false @@ -718,17 +659,17 @@ extension Instruction { move(before: terminator, context) } - if let singleValueInst = self as? SingleValueInstruction, - !(self is BeginAccessInst), - let identicalInst = (preheader.instructions.first { otherInst in - return singleValueInst != otherInst && singleValueInst.isIdenticalTo(otherInst) - }) { - guard let identicalSingleValueInst = identicalInst as? SingleValueInstruction else { - return true - } - - singleValueInst.replace(with: identicalSingleValueInst, context) - } +// if let singleValueInst = self as? SingleValueInstruction, +// !(self is BeginAccessInst), +// let identicalInst = (preheader.instructions.first { otherInst in +// return singleValueInst != otherInst && singleValueInst.isIdenticalTo(otherInst) +// }) { +// guard let identicalSingleValueInst = identicalInst as? SingleValueInstruction else { +// return true +// } +// +// singleValueInst.replace(with: identicalSingleValueInst, context) +// } return true } @@ -794,11 +735,11 @@ extension Instruction { } fileprivate func globalInitMayConflictWith( - loopSideEffects: Stack, + loopSideEffects: StackWithCount, blockSideEffectBottomMarker: Stack.Marker, aliasAnalysis: AliasAnalysis ) -> Bool { - return Stack.Segment(in: loopSideEffects, low: blockSideEffectBottomMarker, high: loopSideEffects.top) + return StackWithCount.Segment(in: loopSideEffects, low: blockSideEffectBottomMarker, high: loopSideEffects.top) .contains { sideEffect in globalInitMayConflictWith( sideEffect: sideEffect, @@ -809,7 +750,7 @@ extension Instruction { fileprivate func globalInitMayConflictWith( preheader: BasicBlock, - loopSideEffects: Stack, + loopSideEffects: StackWithCount, aliasAnalysis: AliasAnalysis, postDomTree: PostDominatorTree ) -> Bool { @@ -832,7 +773,7 @@ extension Instruction { extension UnaryInstruction { fileprivate func mayWriteTo( - sideEffects: Stack, + sideEffects: StackWithCount, aliasAnalysis: AliasAnalysis ) -> Bool { return sideEffects @@ -920,7 +861,7 @@ extension ApplyInst { } fileprivate func mayWriteTo( - sideEffects: Stack, + sideEffects: StackWithCount, aliasAnalysis: AliasAnalysis, calleeAnalysis: CalleeAnalysis ) -> Bool { diff --git a/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift b/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift index 402d5b887e9a6..30444bdb7723b 100644 --- a/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift +++ b/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift @@ -219,3 +219,87 @@ public extension Stack { } } } + +public struct StackWithCount : CollectionLikeSequence { + public private(set) var count = 0 + private var underlyingStack: Stack + + public typealias Iterator = Stack.Iterator + + public init(_ context: some Context) { + self.underlyingStack = Stack(context) + } + + public func makeIterator() -> Stack.Iterator { + underlyingStack.makeIterator() + } + + public var first: Element? { underlyingStack.first } + public var last: Element? { underlyingStack.last } + + public mutating func push(_ element: Element) { + count += 1 + underlyingStack.push(element) + } + + /// The same as `push` to provide an Array-like append API. + public mutating func append(_ element: Element) { push(element) } + + public mutating func append(contentsOf other: S) where S.Element == Element { + for elem in other { + append(elem) + } + } + + public var isEmpty: Bool { underlyingStack.isEmpty } + + public mutating func pop() -> Element? { + if underlyingStack.isEmpty { + return nil + } + + count -= 1 + return underlyingStack.pop() + } + + public mutating func removeAll() { + underlyingStack.removeAll() + } + + /// TODO: once we have move-only types, make this a real deinit. + public mutating func deinitialize() { removeAll() } +} + +public extension StackWithCount { + typealias Marker = Stack.Marker + + struct Segment : CollectionLikeSequence { + var underlyingSegment: Stack.Segment + + public init(in stack: StackWithCount, low: Marker, high: Marker) { + underlyingSegment = Stack.Segment(in: stack.underlyingStack, low: low, high: high) + } + + public func makeIterator() -> StackWithCount.Iterator { + return underlyingSegment.makeIterator() + } + } + + var top: Marker { underlyingStack.top } + + func assertValid(marker: Marker) { underlyingStack.assertValid(marker: marker) } + + mutating func withMarker( + _ body: (inout Stack, Marker) throws -> R) rethrows -> R { + return try underlyingStack.withMarker(body) + } + + mutating func withMarker( + pushElements body: (inout Stack) throws -> R, + withNewElements handleNewElements: ((Segment) -> ()) + ) rethrows -> R { + return try underlyingStack.withMarker(pushElements: body) { [self] segment in + handleNewElements(Segment(in: self, low: segment.low, high: segment.high)) + } + } +} diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index a72831cfe29c9..404aea3119a38 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -1043,6 +1043,7 @@ struct State { // CHECK: bb0(%0 : $Builtin.RawPointer): // CHECK: [[HOISTADR:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 0 // ...Preload stored element #1 +// CHECK: tuple_element_addr // CHECK: [[PRELOADADR:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 1 // CHECK: [[PRELOAD:%.*]] = load [[PRELOADADR]] : $*Int64 // ...Split element 0 From c95838c70d12bec7715fa49a9e460688084fa94a Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Fri, 8 Aug 2025 18:05:20 +0100 Subject: [PATCH 32/44] Refactoring. --- .../Sources/Optimizer/Analysis/LoopTree.swift | 14 + .../LoopInvariantCodeMotion.swift | 353 ++++++++---------- .../Optimizer/Utilities/OptUtils.swift | 16 + 3 files changed, 184 insertions(+), 199 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index 5ff516c3051df..3ca48e0b15e41 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -102,6 +102,20 @@ struct Loop { ) } } + + func splitCriticalEdges(_ context: FunctionPassContext) { + for exitingOrLatchBlock in exitingAndLatchBlocks { + for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !contains(block: succesor) { + splitCriticalEdge( + from: exitingOrLatchBlock.terminator.parentBlock, + toEdgeIndex: index, + dominatorTree: context.dominatorTree, + loopTree: context.loopTree, + context + ) + } + } + } } struct TopLevelLoopArray: BridgedRandomAccessCollection { diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index a1771c2f8ebde..6fe2be435283c 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -44,7 +44,7 @@ private func getWorkList(forLoop loop: Loop, result: inout Stack) { } } -struct MovableInstructions { +private struct MovableInstructions { var loadAndStoreAddrs: [AccessPath] = [] var simplyHoistableInsts: [Instruction] = [] @@ -54,10 +54,18 @@ struct MovableInstructions { var scopedInsts: [Instruction] = [] } -struct AnalyzedInstructions { - var globalInitCalls: Stack +private struct AnalyzedInstructions { var loopSideEffects: StackWithCount + private var blockSideEffectBottomMarker: StackWithCount.Marker + var sideEffectsOfCurrentBlock: StackWithCount.Segment { + return StackWithCount.Segment( + in: loopSideEffects, + low: blockSideEffectBottomMarker, + high: loopSideEffects.top + ) + } + var globalInitCalls: Stack var readOnlyApplies: StackWithCount var loads: StackWithCount var stores: Stack @@ -69,9 +77,10 @@ struct AnalyzedInstructions { lazy var sideEffectsMayRelease = loopSideEffects.contains(where: { $0.mayRelease }) init (_ context: FunctionPassContext) { - self.globalInitCalls = Stack(context) self.loopSideEffects = StackWithCount(context) + self.blockSideEffectBottomMarker = loopSideEffects.top + self.globalInitCalls = Stack(context) self.readOnlyApplies = StackWithCount(context) self.loads = StackWithCount(context) self.stores = Stack(context) @@ -88,20 +97,26 @@ struct AnalyzedInstructions { beginAccesses.deinitialize() fullApplies.deinitialize() } + + mutating func markBeginOfBlock() { + blockSideEffectBottomMarker = loopSideEffects.top + } } private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext) -> MovableInstructions? { guard let preheader = loop.preheader else { return nil } + + loop.splitCriticalEdges(context) var movableInstructions = MovableInstructions() var analyzedInstructions = AnalyzedInstructions(context) defer { analyzedInstructions.deinitialize() } for bb in loop.loopBlocks { - let blockSideEffectBottomMarker = analyzedInstructions.loopSideEffects.top - + analyzedInstructions.markBeginOfBlock() + for inst in bb.instructions { // TODO: Remove once support for values with ownership implemented. if inst.hasOwnershipOperandsOrResults { @@ -134,16 +149,13 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext case let condFailInst as CondFailInst: analyzedInstructions.analyzeSideEffects(ofInst: condFailInst) case let applyInst as ApplyInst: - if applyInst.isSafeReadOnlyApply( - calleeAnalysis: context.calleeAnalysis - ) { + if applyInst.isSafeReadOnlyApply(context.calleeAnalysis) { analyzedInstructions.readOnlyApplies.append(applyInst) } else if let callee = applyInst.referencedFunction, callee.isGlobalInitFunction, !applyInst.globalInitMayConflictWith( - loopSideEffects: analyzedInstructions.loopSideEffects, - blockSideEffectBottomMarker: blockSideEffectBottomMarker, - aliasAnalysis: context.aliasAnalysis + blockSideEffectSegment: analyzedInstructions.sideEffectsOfCurrentBlock, + context.aliasAnalysis ) { analyzedInstructions.globalInitCalls.append(applyInst) } @@ -157,9 +169,8 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext switch builtinInst.id { case .Once, .OnceWithContext: if !builtinInst.globalInitMayConflictWith( - loopSideEffects: analyzedInstructions.loopSideEffects, - blockSideEffectBottomMarker: blockSideEffectBottomMarker, - aliasAnalysis: context.aliasAnalysis + blockSideEffectSegment: analyzedInstructions.sideEffectsOfCurrentBlock, + context.aliasAnalysis ) { analyzedInstructions.globalInitCalls.append(builtinInst) } @@ -180,10 +191,10 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. if analyzedInstructions.readOnlyApplies.count * analyzedInstructions.loopSideEffects.count < 8000 { for readOnlyApply in analyzedInstructions.readOnlyApplies { - if !readOnlyApply.mayWriteTo( - sideEffects: analyzedInstructions.loopSideEffects, - aliasAnalysis: context.aliasAnalysis, - calleeAnalysis: context.calleeAnalysis + if !readOnlyApply.isSafeReadOnlyApply( + for: analyzedInstructions.loopSideEffects, + context.aliasAnalysis, + context.calleeAnalysis ) { movableInstructions.hoistUp.append(readOnlyApply) } @@ -192,10 +203,10 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext for globalInitCall in analyzedInstructions.globalInitCalls { if !globalInitCall.globalInitMayConflictWith( - preheader: preheader, loopSideEffects: analyzedInstructions.loopSideEffects, - aliasAnalysis: context.aliasAnalysis, - postDomTree: context.postDominatorTree + notPostDominatingPreheader: preheader, + context.aliasAnalysis, + context.postDominatorTree ) { movableInstructions.hoistUp.append(globalInitCall) } @@ -208,9 +219,10 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext accessPath.isOnlyLoadedAndStored( storeAddr: storeInst.destination, analyzedInstructions: analyzedInstructions, - aliasAnalysis: context.aliasAnalysis + context.aliasAnalysis ), !movableInstructions.loadAndStoreAddrs.contains(accessPath), + loop.storesCommonlyDominateExits(analyzedInstructions: analyzedInstructions, accessPath: accessPath, context), analyzedInstructions.splitLoads( storeAddr: storeInst.destination, accessPath: accessPath, @@ -236,14 +248,14 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext if analyzedInstructions.sideEffectsMayRelease || !fixLifetimeInst.mayWriteTo( sideEffects: analyzedInstructions.loopSideEffects, - aliasAnalysis: context.aliasAnalysis + context.aliasAnalysis ) { movableInstructions.sinkDown.append(fixLifetimeInst) } case let loadInst as LoadInst: // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. if analyzedInstructions.loads.count * analyzedInstructions.loopSideEffects.count < 8000, - !loadInst.mayWriteTo(sideEffects: analyzedInstructions.loopSideEffects, aliasAnalysis: context.aliasAnalysis) { + !loadInst.mayWriteTo(sideEffects: analyzedInstructions.loopSideEffects, context.aliasAnalysis) { movableInstructions.hoistUp.append(loadInst) } @@ -259,12 +271,7 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext case let condFailInst as CondFailInst: movableInstructions.hoistUp.append(condFailInst) case let beginAccessInst as BeginAccessInst: - if beginAccessInst.canBeHoisted( - outOf: loop, - analyzedInstructions: analyzedInstructions, - aliasAnalysis: context.aliasAnalysis, - domTree: context.dominatorTree - ) { + if beginAccessInst.canBeHoisted(outOf: loop, analyzedInstructions: analyzedInstructions, context) { movableInstructions.scopedInsts.append(beginAccessInst) } default: @@ -293,7 +300,7 @@ private func optimizeLoop( return changed } -extension AnalyzedInstructions { +private extension AnalyzedInstructions { mutating func analyzeSideEffects(ofInst inst: Instruction) { if inst.mayHaveSideEffects { loopSideEffects.append(inst) @@ -335,7 +342,7 @@ extension AnalyzedInstructions { } } -extension MovableInstructions { +private extension MovableInstructions { mutating func simpleHoistInstructions(loop: Loop, _ context: FunctionPassContext) -> Bool { var changed = false for inst in simplyHoistableInsts { @@ -394,7 +401,7 @@ extension MovableInstructions { } for endAccess in beginAccessInst.endAccessInstructions { - endAccess.sink(outOf: loop, context) + _ = endAccess.sink(outOf: loop, context) } changed = true @@ -410,22 +417,6 @@ extension MovableInstructions { ) -> Bool { let exitingAndLatchBlocks = loop.exitingAndLatchBlocks - guard loop.storesCommonlyDominateExits(accessPath: accessPath, context) else { - return false - } - - for exitingOrLatchBlock in exitingAndLatchBlocks { - for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !loop.contains(block: succesor) { - splitCriticalEdge( - from: exitingOrLatchBlock.terminator.parentBlock, - toEdgeIndex: index, - dominatorTree: context.dominatorTree, - loopTree: context.loopTree, - context - ) - } - } - guard let preheader = loop.preheader else { return false } @@ -435,9 +426,9 @@ extension MovableInstructions { var ssaUpdater: SSAUpdater? var storeAddr: Value? - for case let storeInst as StoreInst in loadsAndStores where storeInst.accesses(accessPath) { + for case let storeInst as StoreInst in loadsAndStores where storeInst.storesTo(accessPath) { if let srcLoadInst = storeInst.source as? LoadInst, - srcLoadInst.accesses(accessPath) { + srcLoadInst.loadsFrom(accessPath) { return changed } @@ -487,16 +478,15 @@ extension MovableInstructions { currentVal = nil } - if let storeInst = inst as? StoreInst, storeInst.accesses(accessPath) { + if let storeInst = inst as? StoreInst, storeInst.storesTo(accessPath) { currentVal = storeInst.source - loadsAndStores.removeAll(where: { $0 == storeInst }) context.erase(instruction: storeInst) changed = true continue } guard let loadInst = inst as? LoadInst, - loadInst.accesses(accessPath) else { + loadInst.loadsFrom(accessPath) else { continue } @@ -511,10 +501,11 @@ extension MovableInstructions { continue } - loadsAndStores.removeAll(where: { $0 == loadInst }) changed = true } + loadsAndStores.removeAll(where: { $0.isDeleted }) + for exitingOrLatchBlock in exitingAndLatchBlocks { for succesor in exitingOrLatchBlock.successors where !loop.contains(block: succesor) { guard let firstInst = succesor.instructions.first else { @@ -533,7 +524,7 @@ extension MovableInstructions { } } - if initialLoad.isTriviallyDead || initialLoad.uses.isEmpty { + if initialLoad.uses.isEmpty { context.erase(instruction: initialLoad) } @@ -541,81 +532,59 @@ extension MovableInstructions { } } -extension Loop { - fileprivate func storesCommonlyDominateExits(accessPath: AccessPath, _ context: FunctionPassContext) -> Bool { - var stores = BasicBlockSet(context) - var storesNotAlive = BasicBlockSet(context) - var uses = Stack(context) +private extension Loop { + func storesCommonlyDominateExits( + analyzedInstructions: AnalyzedInstructions, + accessPath: AccessPath, + _ context: FunctionPassContext + ) -> Bool { + var storeBlocks = BasicBlockSet(context) + var worklist = BasicBlockWorklist(context) defer { - stores.deinitialize() - storesNotAlive.deinitialize() - uses.deinitialize() + storeBlocks.deinitialize() + worklist.deinitialize() } - for use in header.parentFunction.instructions.flatMap(\.operands) where use.value.accessPath == accessPath { - if let user = use.instruction as? StoreInst { - stores.insert(user.parentBlock) - } + if preheader!.instructions.contains(where: { $0.operands.contains(where: { operand in + operand.instruction is StoreInst && operand.value.accessPath == accessPath + })}) { + storeBlocks.insert(preheader!) } - guard !stores.contains(header) else { - return true - } + storeBlocks.insert(contentsOf: analyzedInstructions.stores + .filter({ $0.destination.accessPath == accessPath }) + .map(\.parentBlock)) - if let preheader = preheader, - stores.contains(preheader) { + if storeBlocks.contains(preheader!) || storeBlocks.contains(header) { return true } - storesNotAlive.insert(header) - - var changed = false - repeat { - changed = false - for block in loopBlocks where !storesNotAlive.contains(block) && !stores.contains(block) && block.predecessors.contains(where: { storesNotAlive.contains($0) }) { - storesNotAlive.insert(block) - changed = true + worklist.pushIfNotVisited(header) + while let block = worklist.pop() { + if storeBlocks.contains(block) { + continue } - } while changed - - for exitingBlock in exitingBlocks where !exitingBlock.successors.contains(where: { $0.terminator is UnreachableInst }) && storesNotAlive.contains(exitingBlock) { - return false + + if exitingBlocks.contains(block) && !block.successors.contains(where: {$0.terminator is UnreachableInst}) { + return false + } + + worklist.pushIfNotVisited(contentsOf: block.successors) } return true } } -extension Instruction { - fileprivate func dominates( - _ otherInst: Instruction, - _ domTree: DominatorTree - ) -> Bool { - if parentBlock == otherInst.parentBlock { - for inst in parentBlock.instructions { - if inst == self { - return true - } else if inst == otherInst { - break - } - } - return false - } else { - return parentBlock.dominates( - otherInst.parentBlock, - domTree - ) - } - } - - fileprivate var hasOwnershipOperandsOrResults: Bool { +private extension Instruction { + var hasOwnershipOperandsOrResults: Bool { guard parentFunction.hasOwnership else { return false } return results.contains(where: { $0.ownership != .none }) || operands.contains(where: { $0.value.ownership != .none }) } - fileprivate func canBeHoisted(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + func canBeHoisted(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { guard let preheader = loop.preheader else { return false } @@ -645,8 +614,7 @@ extension Instruction { return false } - @discardableResult - fileprivate func hoist(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + func hoist(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { guard let preheader = loop.preheader, operands.allSatisfy({ !loop.contains(block: $0.value.parentBlock) }) else { return false @@ -659,24 +627,23 @@ extension Instruction { move(before: terminator, context) } -// if let singleValueInst = self as? SingleValueInstruction, -// !(self is BeginAccessInst), -// let identicalInst = (preheader.instructions.first { otherInst in -// return singleValueInst != otherInst && singleValueInst.isIdenticalTo(otherInst) -// }) { -// guard let identicalSingleValueInst = identicalInst as? SingleValueInstruction else { -// return true -// } -// -// singleValueInst.replace(with: identicalSingleValueInst, context) -// } + if let singleValueInst = self as? SingleValueInstruction, + !(self is BeginAccessInst), + let identicalInst = (preheader.instructions.first { otherInst in + return singleValueInst != otherInst && singleValueInst.isIdenticalTo(otherInst) + }) { + guard let identicalSingleValueInst = identicalInst as? SingleValueInstruction else { + return true + } + + singleValueInst.replace(with: identicalSingleValueInst, context) + } return true } - @discardableResult - fileprivate func sink(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { - var isSingleExit = loop.isSingleExit + func sink(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + var canMove = loop.isSingleExit let exitBlocks = loop.exitBlocks let exitingBlocks = loop.exitingBlocks var newExitBlocks = Stack(context) @@ -684,24 +651,15 @@ extension Instruction { var changed = false for exitingBlock in exitingBlocks { - for (succesorIndex, succesor) in exitingBlock.successors.enumerated().reversed() - where !newExitBlocks.contains(succesor) && exitBlocks.contains(succesor) { - - let outsideBlock = splitCriticalEdge( - from: exitingBlock, - toEdgeIndex: succesorIndex, - dominatorTree: context.dominatorTree, - loopTree: context.loopTree, - context - ) ?? succesor + for succesor in exitingBlock.successors where !newExitBlocks.contains(succesor) && exitBlocks.contains(succesor) { - newExitBlocks.push(outsideBlock) + newExitBlocks.push(succesor) - if outsideBlock.instructions.contains(where: { isIdenticalTo($0) }) { - isSingleExit = false - } else if isSingleExit, let firstInstruction = outsideBlock.instructions.first { - move(before: firstInstruction, context) - } else if let firstInstruction = outsideBlock.instructions.first { + if succesor.instructions.contains(where: { isIdenticalTo($0) }) { + canMove = false + } else if canMove { + move(before: succesor.instructions.first!, context) + } else if let firstInstruction = succesor.instructions.first { copy(before: firstInstruction, context) } else { continue @@ -711,7 +669,7 @@ extension Instruction { } } - if changed && !isSingleExit { + if changed && !canMove { context.erase(instruction: self) } @@ -720,7 +678,7 @@ extension Instruction { private func globalInitMayConflictWith( sideEffect: Instruction, - aliasAnalysis: AliasAnalysis + _ aliasAnalysis: AliasAnalysis ) -> Bool { switch sideEffect { case let storeInst as StoreInst: @@ -734,27 +692,26 @@ extension Instruction { } } - fileprivate func globalInitMayConflictWith( - loopSideEffects: StackWithCount, - blockSideEffectBottomMarker: Stack.Marker, - aliasAnalysis: AliasAnalysis + func globalInitMayConflictWith( + blockSideEffectSegment: StackWithCount.Segment, + _ aliasAnalysis: AliasAnalysis ) -> Bool { - return StackWithCount.Segment(in: loopSideEffects, low: blockSideEffectBottomMarker, high: loopSideEffects.top) + return blockSideEffectSegment .contains { sideEffect in globalInitMayConflictWith( sideEffect: sideEffect, - aliasAnalysis: aliasAnalysis + aliasAnalysis ) } } - fileprivate func globalInitMayConflictWith( - preheader: BasicBlock, + func globalInitMayConflictWith( loopSideEffects: StackWithCount, - aliasAnalysis: AliasAnalysis, - postDomTree: PostDominatorTree + notPostDominatingPreheader: BasicBlock, + _ aliasAnalysis: AliasAnalysis, + _ postDomTree: PostDominatorTree ) -> Bool { - guard parentBlock.postDominates(preheader, postDomTree) else { + guard parentBlock.postDominates(notPostDominatingPreheader, postDomTree) else { return true } @@ -765,16 +722,23 @@ extension Instruction { postDomTree ) && globalInitMayConflictWith( sideEffect: sideEffect, - aliasAnalysis: aliasAnalysis + aliasAnalysis ) } } + + func isInAccessScope(of beginAccess: BeginAccessInst, _ domTree: DominatorTree) -> Bool { + return beginAccess.dominates(self, domTree) && beginAccess.endAccessInstructions + .allSatisfy { endAccessInst in + self.dominates(endAccessInst, domTree) + } + } } -extension UnaryInstruction { - fileprivate func mayWriteTo( +private extension UnaryInstruction { + func mayWriteTo( sideEffects: StackWithCount, - aliasAnalysis: AliasAnalysis + _ aliasAnalysis: AliasAnalysis ) -> Bool { return sideEffects .contains { sideEffect in @@ -783,8 +747,8 @@ extension UnaryInstruction { } } -extension StoreInst { - fileprivate func accesses(_ accessPath: AccessPath) -> Bool { +private extension StoreInst { + func storesTo(_ accessPath: AccessPath) -> Bool { guard self.storeOwnership != .initialize else { return false } @@ -793,8 +757,8 @@ extension StoreInst { } } -extension LoadInst { - fileprivate func replaceLoadWithProjection( +private extension LoadInst { + func replaceLoadWithProjection( rootVal: Value, rootAccessPath: AccessPath, beforeInst: Instruction, @@ -833,11 +797,11 @@ extension LoadInst { return true } - fileprivate func accesses(_ accessPath: AccessPath) -> Bool { + func loadsFrom(_ accessPath: AccessPath) -> Bool { return accessPath.getProjection(to: self.address.accessPath)?.isMaterializable ?? false } - fileprivate func overlaps(accessPath: AccessPath) -> Bool { + func overlaps(accessPath: AccessPath) -> Bool { guard self.loadOwnership != .take else { // TODO: handle LoadOwnershipQualifier::Take return false } @@ -846,8 +810,8 @@ extension LoadInst { } } -extension ApplyInst { - fileprivate func isSafeReadOnlyApply(calleeAnalysis: CalleeAnalysis) -> Bool { +private extension ApplyInst { + func isSafeReadOnlyApply(_ calleeAnalysis: CalleeAnalysis) -> Bool { guard functionConvention.results.allSatisfy({ $0.convention == .unowned }) else { return false } @@ -860,12 +824,12 @@ extension ApplyInst { return calleeAnalysis.getSideEffects(ofApply: self).isOnlyReading } - fileprivate func mayWriteTo( - sideEffects: StackWithCount, - aliasAnalysis: AliasAnalysis, - calleeAnalysis: CalleeAnalysis + func isSafeReadOnlyApply( + for sideEffects: StackWithCount, + _ aliasAnalysis: AliasAnalysis, + _ calleeAnalysis: CalleeAnalysis ) -> Bool { - guard calleeAnalysis.getSideEffects(ofApply: self).memory != .noEffects else { + if calleeAnalysis.getSideEffects(ofApply: self).memory == .noEffects { return false } @@ -901,14 +865,7 @@ extension ApplyInst { } } -extension BeginAccessInst { - private func isCoveredByScope(otherInst: Instruction, domTree: DominatorTree) -> Bool { - return self.dominates(otherInst, domTree) && endAccessInstructions - .allSatisfy { endAccessInst in - otherInst.dominates(endAccessInst, domTree) - } - } - +private extension BeginAccessInst { private func handlesAllEndAccesses(loop: Loop) -> Bool { return !endAccessInstructions.isEmpty && !endAccessInstructions .contains { user in @@ -916,11 +873,10 @@ extension BeginAccessInst { } } - fileprivate func canBeHoisted( + func canBeHoisted( outOf loop: Loop, analyzedInstructions: AnalyzedInstructions, - aliasAnalysis: AliasAnalysis, - domTree: DominatorTree + _ context: FunctionPassContext ) -> Bool { guard handlesAllEndAccesses(loop: loop) else { return false @@ -934,32 +890,31 @@ extension BeginAccessInst { } guard areBeginAccessesSafe else { return false } + + var scope = InstructionRange(begin: self, ends: endAccessInstructions, context) + defer { scope.deinitialize() } for fullApplyInst in analyzedInstructions.fullApplies { guard mayWriteToMemory ? fullApplyInst.mayReadOrWrite( address: address, - aliasAnalysis + context.aliasAnalysis ) : fullApplyInst.mayWrite( toAddress: address, - aliasAnalysis + context.aliasAnalysis ) else { continue } - if !isCoveredByScope(otherInst: fullApplyInst, domTree: domTree) { + if !scope.contains(fullApplyInst) { return false } } switch accessPath.base { case .class, .global: - for sideEffect in analyzedInstructions.loopSideEffects { - guard sideEffect.mayRelease else { - continue - } - - if !isCoveredByScope(otherInst: sideEffect, domTree: domTree) { + for sideEffect in analyzedInstructions.loopSideEffects where sideEffect.mayRelease { + if !scope.contains(sideEffect) { return false } } @@ -971,8 +926,8 @@ extension BeginAccessInst { } } -extension AccessPath { - fileprivate func isLoopInvariant(loop: Loop) -> Bool { +private extension AccessPath { + func isLoopInvariant(loop: Loop) -> Bool { switch base { case .box(let inst as Instruction), .class(let inst as Instruction), .index(let inst as Instruction), @@ -995,19 +950,19 @@ extension AccessPath { return projectionPath.isConstant } - fileprivate func isOnlyLoadedAndStored( + func isOnlyLoadedAndStored( storeAddr: Value, analyzedInstructions: AnalyzedInstructions, - aliasAnalysis: AliasAnalysis + _ aliasAnalysis: AliasAnalysis ) -> Bool { return !analyzedInstructions.loopSideEffects .contains { sideEffect in let accessesPath: Bool switch sideEffect { case let storeInst as StoreInst: - accessesPath = storeInst.accesses(self) + accessesPath = storeInst.storesTo(self) case let loadInst as LoadInst: - accessesPath = loadInst.accesses(self) + accessesPath = loadInst.loadsFrom(self) default: accessesPath = false } @@ -1020,7 +975,7 @@ extension AccessPath { } && !analyzedInstructions.stores .contains { storeInst in storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) - && !storeInst.accesses(self) + && !storeInst.storesTo(self) } } } diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift index 85bd2a2849d5c..443959301770f 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift @@ -485,6 +485,22 @@ extension Instruction { } return false } + + /// Returns true if `otherInst` is in the same block and is strictly dominated by this instruction or + /// the parent block of the instruction dominates parent block of `otherInst`. + func dominates( + _ otherInst: Instruction, + _ domTree: DominatorTree + ) -> Bool { + if parentBlock == otherInst.parentBlock { + return dominatesInSameBlock(otherInst) + } else { + return parentBlock.dominates( + otherInst.parentBlock, + domTree + ) + } + } /// If this instruction uses a (single) existential archetype, i.e. it has a type-dependent operand, /// returns the concrete type if it is known. From f23e65c00c8274d42f5cae9600cd3a488fb1924f Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Tue, 12 Aug 2025 16:07:42 +0100 Subject: [PATCH 33/44] Fix read only apply handling. Refactoring. --- .../LoopInvariantCodeMotion.swift | 80 ++++++++++--------- .../Sources/Optimizer/Utilities/Cloner.swift | 7 -- .../Sources/SIL/Utilities/AccessUtils.swift | 1 + .../swift/SILOptimizer/OptimizerBridging.h | 11 --- .../SILOptimizer/OptimizerBridgingImpl.h | 35 -------- .../BuildArgs/ClangBuildArgsProvider.swift | 2 +- .../Command/CompileCommands.swift | 2 +- 7 files changed, 45 insertions(+), 93 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 6fe2be435283c..3f37d90fc9532 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -191,7 +191,7 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. if analyzedInstructions.readOnlyApplies.count * analyzedInstructions.loopSideEffects.count < 8000 { for readOnlyApply in analyzedInstructions.readOnlyApplies { - if !readOnlyApply.isSafeReadOnlyApply( + if readOnlyApply.isSafeReadOnlyApply( for: analyzedInstructions.loopSideEffects, context.aliasAnalysis, context.calleeAnalysis @@ -548,14 +548,14 @@ private extension Loop { if preheader!.instructions.contains(where: { $0.operands.contains(where: { operand in operand.instruction is StoreInst && operand.value.accessPath == accessPath })}) { - storeBlocks.insert(preheader!) + return true } storeBlocks.insert(contentsOf: analyzedInstructions.stores .filter({ $0.destination.accessPath == accessPath }) .map(\.parentBlock)) - if storeBlocks.contains(preheader!) || storeBlocks.contains(header) { + if storeBlocks.contains(header) { return true } @@ -821,7 +821,7 @@ private extension ApplyInst { return false } - return calleeAnalysis.getSideEffects(ofApply: self).isOnlyReading + return !calleeAnalysis.getSideEffects(ofApply: self).memory.write } func isSafeReadOnlyApply( @@ -830,7 +830,7 @@ private extension ApplyInst { _ calleeAnalysis: CalleeAnalysis ) -> Bool { if calleeAnalysis.getSideEffects(ofApply: self).memory == .noEffects { - return false + return true } for sideEffect in sideEffects { @@ -838,16 +838,16 @@ private extension ApplyInst { case let storeInst as StoreInst: if storeInst.storeOwnership == .assign || mayRead(fromAddress: storeInst.destination, aliasAnalysis) { - return true + return false } case let copyAddrInst as CopyAddrInst: if !copyAddrInst.isInitializationOfDestination || mayRead(fromAddress: copyAddrInst.destination, aliasAnalysis) { - return true + return false } case is ApplyInst, is BeginApplyInst, is TryApplyInst: - if !calleeAnalysis.getSideEffects(ofApply: self).isOnlyReading { - return true + if calleeAnalysis.getSideEffects(ofApply: self).memory.write { + return false } case is CondFailInst, is StrongRetainInst, is UnmanagedRetainValueInst, is RetainValueInst, is StrongRetainUnownedInst, is FixLifetimeInst, @@ -856,21 +856,18 @@ private extension ApplyInst { break default: if sideEffect.mayWriteToMemory { - return true + return false } } } - return false + return true } } private extension BeginAccessInst { - private func handlesAllEndAccesses(loop: Loop) -> Bool { - return !endAccessInstructions.isEmpty && !endAccessInstructions - .contains { user in - !loop.contains(block: user.parentBlock) - } + private func areAllEndAccesses(in loop: Loop) -> Bool { + return endAccessInstructions.allSatisfy({ loop.contains(block: $0.parentBlock)}) } func canBeHoisted( @@ -878,7 +875,7 @@ private extension BeginAccessInst { analyzedInstructions: AnalyzedInstructions, _ context: FunctionPassContext ) -> Bool { - guard handlesAllEndAccesses(loop: loop) else { + guard areAllEndAccesses(in: loop) else { return false } @@ -955,27 +952,34 @@ private extension AccessPath { analyzedInstructions: AnalyzedInstructions, _ aliasAnalysis: AliasAnalysis ) -> Bool { - return !analyzedInstructions.loopSideEffects - .contains { sideEffect in - let accessesPath: Bool - switch sideEffect { - case let storeInst as StoreInst: - accessesPath = storeInst.storesTo(self) - case let loadInst as LoadInst: - accessesPath = loadInst.loadsFrom(self) - default: - accessesPath = false - } - - return !accessesPath && sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) - } && !analyzedInstructions.loads - .contains { loadInst in - loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) - && !loadInst.overlaps(accessPath: self) - } && !analyzedInstructions.stores - .contains { storeInst in - storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) - && !storeInst.storesTo(self) + if (analyzedInstructions.loopSideEffects.contains { sideEffect in + let accessesPath: Bool + switch sideEffect { + case let storeInst as StoreInst: + accessesPath = storeInst.storesTo(self) + case let loadInst as LoadInst: + accessesPath = loadInst.loadsFrom(self) + default: + accessesPath = false } + + return !accessesPath && sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) + }) { + return false + } + + if (analyzedInstructions.loads.contains { loadInst in + loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) && !loadInst.overlaps(accessPath: self) + }) { + return false + } + + if (analyzedInstructions.stores.contains { storeInst in + storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && !storeInst.storesTo(self) + }) { + return false + } + + return true } } diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift index 3d66618555de5..dd739bdfe5b96 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift @@ -121,13 +121,6 @@ struct Cloner { } } - private func shouldClone(_ value: Value) -> Bool { - switch value { - case is StructElementAddrInst, is TupleElementAddrInst, is IndexAddrInst, is TailAddrInst, is InitEnumDataAddrInst, is OpenExistentialAddrInst, is UncheckedTakeEnumDataAddrInst, is ProjectBoxInst, is ProjectBlockStorageInst, is MoveOnlyWrapperToCopyableAddrInst, is CopyableToMoveOnlyWrapperAddrInst, is MoveOnlyWrapperToCopyableBoxInst, is UncheckedAddrCastInst, is AddressToPointerInst, is PointerToAddressInst, is MarkUninitializedInst, is MarkUnresolvedReferenceBindingInst, is DropDeinitInst, is MarkUnresolvedReferenceBindingInst, is MarkDependenceInst, is CopyValueInst, is BeginBorrowInst, is StoreBorrowInst: return true - default: return false - } - } - private mutating func cloneProjection( projectAddr: SingleValueInstruction, sourceOperand: Operand, diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift b/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift index 8899046e18c0c..d0863f9b3d78d 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift @@ -393,6 +393,7 @@ public struct AccessPath : CustomStringConvertible, Hashable { return getProjection(to: other) != nil } + /// Returns true if this access contains `other` access and is not equal. public func contains(_ other: AccessPath) -> Bool { return !(getProjection(to: other)?.isEmpty ?? true) } diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index 7834f5776fa40..0a069371acb0b 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -45,9 +45,6 @@ class DominanceInfo; class PostDominanceInfo; class SILLoopInfo; class SILLoop; -class BasicBlockSet; -class NodeSet; -class OperandSet; class BridgedClonerImpl; class SwiftPassInvocation; class SILVTable; @@ -195,14 +192,6 @@ struct BridgedPassContext { bool cfgChanged; }; - SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedBasicBlock splitBlockBefore(BridgedInstruction bridgedInst) const; - SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedBasicBlock splitBlockAfter(BridgedInstruction bridgedInst) const; - SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedBasicBlock createBlockAfter(BridgedBasicBlock bridgedBlock) const; - SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedBasicBlock appendBlock(BridgedFunction bridgedFunction) const; - BRIDGED_INLINE void eraseInstruction(BridgedInstruction inst, bool salvageDebugInfo) const; - BRIDGED_INLINE void eraseBlock(BridgedBasicBlock block) const; - static BRIDGED_INLINE void moveInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst); - static BRIDGED_INLINE void copyInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst); bool tryOptimizeApplyOfPartialApply(BridgedInstruction closure) const; bool tryDeleteDeadClosure(BridgedInstruction closure, bool needKeepArgsAlive) const; SWIFT_IMPORT_UNSAFE DevirtResult tryDevirtualizeApply(BridgedInstruction apply, bool isMandatory) const; diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index cb7a5b1458cdb..4fe2d5e3d4553 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -232,41 +232,6 @@ BridgedDiagnosticEngine BridgedPassContext::getDiagnosticEngine() const { // SIL modifications -BridgedBasicBlock BridgedPassContext::splitBlockBefore(BridgedInstruction bridgedInst) const { - auto *block = bridgedInst.unbridged()->getParent(); - return {block->split(bridgedInst.unbridged()->getIterator())}; -} - -BridgedBasicBlock BridgedPassContext::splitBlockAfter(BridgedInstruction bridgedInst) const { - auto *block = bridgedInst.unbridged()->getParent(); - return {block->split(std::next(bridgedInst.unbridged()->getIterator()))}; -} - -BridgedBasicBlock BridgedPassContext::createBlockAfter(BridgedBasicBlock bridgedBlock) const { - swift::SILBasicBlock *block = bridgedBlock.unbridged(); - return {block->getParent()->createBasicBlockAfter(block)}; -} - -BridgedBasicBlock BridgedPassContext::appendBlock(BridgedFunction bridgedFunction) const { - return {bridgedFunction.getFunction()->createBasicBlock()}; -} - -void BridgedPassContext::eraseInstruction(BridgedInstruction inst, bool salvageDebugInfo) const { - invocation->eraseInstruction(inst.unbridged(), salvageDebugInfo); -} - -void BridgedPassContext::eraseBlock(BridgedBasicBlock block) const { - block.unbridged()->eraseFromParent(); -} - -void BridgedPassContext::moveInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst) { - swift::SILBasicBlock::moveInstruction(inst.unbridged(), beforeInst.unbridged()); -} - -void BridgedPassContext::copyInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst) { - inst.unbridged()->clone(beforeInst.unbridged()); -} - bool BridgedPassContext::eliminateDeadAllocations(BridgedFunction f) const { return swift::eliminateDeadAllocations(f.getFunction(), this->getDomTree().di); diff --git a/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/ClangBuildArgsProvider.swift b/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/ClangBuildArgsProvider.swift index a226e59499a83..4d9075406ca64 100644 --- a/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/ClangBuildArgsProvider.swift +++ b/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/ClangBuildArgsProvider.swift @@ -42,7 +42,7 @@ struct ClangBuildArgsProvider { else { continue } - let output = command.output.map { command.directory.appending($0) } + let output = command.output.map { $0.absolute(in: command.directory) } if let existing = commandsToAdd[relFilePath], let existingOutput = existing.output, output == nil || existingOutput.exists || !output!.exists { diff --git a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Command/CompileCommands.swift b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Command/CompileCommands.swift index 64b04c73a81c4..09c84e442fb96 100644 --- a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Command/CompileCommands.swift +++ b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Command/CompileCommands.swift @@ -27,7 +27,7 @@ extension CompileCommands { struct Element: Decodable { var directory: AbsolutePath var file: AbsolutePath - var output: RelativePath? + var output: AnyPath? var command: Command } } From 7e9f92c1b9d4500fccf6f58514d61eb9d938a061 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Wed, 13 Aug 2025 16:40:54 +0100 Subject: [PATCH 34/44] Fix handling of FixLifetimeInst. Refactor. --- .../LoopInvariantCodeMotion.swift | 68 +++++-------------- 1 file changed, 16 insertions(+), 52 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 3f37d90fc9532..65b6f007e02bc 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -130,6 +130,8 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext } switch inst { + case is FixLifetimeInst: + break case let loadInst as LoadInst: analyzedInstructions.loads.append(loadInst) case let storeInst as StoreInst: @@ -246,7 +248,7 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext continue } - if analyzedInstructions.sideEffectsMayRelease || !fixLifetimeInst.mayWriteTo( + if !analyzedInstructions.sideEffectsMayRelease || !fixLifetimeInst.mayWriteTo( sideEffects: analyzedInstructions.loopSideEffects, context.aliasAnalysis ) { @@ -491,16 +493,21 @@ private extension MovableInstructions { } let rootVal = currentVal ?? ssaUpdater.getValue(inMiddleOf: block) - - guard loadInst.replaceLoadWithProjection( - rootVal: rootVal, - rootAccessPath: accessPath, - beforeInst: loadInst, - context - ) else { + + if loadInst.operand.value.accessPath == accessPath { + loadInst.replace(with: rootVal, context) + changed = true continue } + guard let projectionPath = accessPath.getProjection(to: loadInst.operand.value.accessPath) else { + continue + } + + let builder = Builder(before: loadInst, context) + let projection = rootVal.createProjection(path: projectionPath, builder: builder) + loadInst.replace(with: projection, context) + changed = true } @@ -758,45 +765,6 @@ private extension StoreInst { } private extension LoadInst { - func replaceLoadWithProjection( - rootVal: Value, - rootAccessPath: AccessPath, - beforeInst: Instruction, - _ context: FunctionPassContext - ) -> Bool { - guard operand.value.accessPath != rootAccessPath else { - replace(with: rootVal, context) - return true - } - - guard let projectionPath = rootAccessPath.getProjection(to: operand.value.accessPath) else { - return false - } - - let builder = Builder(before: beforeInst, context) - - var currPath = projectionPath - var currVal = rootVal - - while !currPath.isEmpty { - let (kind, index, remainderPath) = currPath.pop() - - switch kind { - case .structField: - currVal = builder.createStructExtract(struct: currVal, fieldIndex: index) - case .tupleField: - currVal = builder.createTupleExtract(tuple: currVal, elementIndex: index) - default: - return false - } - - currPath = remainderPath - } - - replace(with: currVal, context) - return true - } - func loadsFrom(_ accessPath: AccessPath) -> Bool { return accessPath.getProjection(to: self.address.accessPath)?.isMaterializable ?? false } @@ -866,16 +834,12 @@ private extension ApplyInst { } private extension BeginAccessInst { - private func areAllEndAccesses(in loop: Loop) -> Bool { - return endAccessInstructions.allSatisfy({ loop.contains(block: $0.parentBlock)}) - } - func canBeHoisted( outOf loop: Loop, analyzedInstructions: AnalyzedInstructions, _ context: FunctionPassContext ) -> Bool { - guard areAllEndAccesses(in: loop) else { + guard endAccessInstructions.allSatisfy({ loop.contains(block: $0.parentBlock)}) else { return false } From 611471d455d248a8b2ab4782999d5ceb6032f692 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Fri, 15 Aug 2025 16:47:54 +0100 Subject: [PATCH 35/44] Minor fixes. Refactoring. --- .../Sources/Optimizer/Analysis/LoopTree.swift | 12 +- .../LoopInvariantCodeMotion.swift | 214 ++++++++++++------ .../Optimizer/PassManager/ContextCommon.swift | 2 +- .../Sources/Optimizer/Utilities/Cloner.swift | 13 +- .../Optimizer/Utilities/OptUtils.swift | 101 ++++----- include/swift/SIL/SILBridging.h | 43 ++-- .../SILOptimizer/Analysis/ArrayCallKind.h | 45 ++++ .../SILOptimizer/Analysis/ArraySemantic.h | 30 +-- .../swift/SILOptimizer/OptimizerBridging.h | 23 +- .../SILOptimizer/OptimizerBridgingImpl.h | 4 +- lib/SIL/Utils/SILBridging.cpp | 140 ++++++++++++ .../LoopTransforms/ForEachLoopUnroll.cpp | 2 +- .../Transforms/DeadObjectElimination.cpp | 2 +- lib/SILOptimizer/Utils/OptimizerBridging.cpp | 136 ----------- test/SILOptimizer/licm_exclusivity.swift | 18 -- 15 files changed, 412 insertions(+), 373 deletions(-) create mode 100644 include/swift/SILOptimizer/Analysis/ArrayCallKind.h diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index 3ca48e0b15e41..59bb3625a80fb 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -32,21 +32,21 @@ struct Loop { let innerLoops: LoopArray let loopBlocks: LoopBlocks - var exitingAndLatchBlocks: [BasicBlock] { - return header.predecessors + var exitingAndLatchBlocks: some Sequence { + return header.predecessors.lazy .filter { predecessor in contains(block: predecessor) && !isLoopExiting(loopBlock: predecessor) } + exitingBlocks } - var exitBlocks: [BasicBlock] { - return loopBlocks + var exitBlocks: some Sequence { + return loopBlocks.lazy .flatMap(\.successors) .filter { !contains(block: $0) } } - var exitingBlocks: [BasicBlock] { - return loopBlocks + var exitingBlocks: some Sequence { + return loopBlocks.lazy .filter { isLoopExiting(loopBlock: $0) } } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 65b6f007e02bc..9ed4ef72dc158 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -26,12 +26,15 @@ private func optimizeTopLevelLoop(topLevelLoop: Loop, _ context: FunctionPassCon getWorkList(forLoop: topLevelLoop, result: &innerLoops) while let thisLoop = innerLoops.pop() { + guard thisLoop.preheader != nil else { + continue + } + var thisLoopChanged = false repeat { - if var movableInstructions = analyzeLoopAndSplitLoads(loop: thisLoop, context) { - thisLoopChanged = optimizeLoop(loop: thisLoop, movableInstructions: &movableInstructions, context) - } + var movableInstructions = analyzeLoopAndSplitLoads(loop: thisLoop, context) + thisLoopChanged = optimizeLoop(loop: thisLoop, movableInstructions: &movableInstructions, context) } while thisLoopChanged } } @@ -47,7 +50,7 @@ private func getWorkList(forLoop loop: Loop, result: inout Stack) { private struct MovableInstructions { var loadAndStoreAddrs: [AccessPath] = [] - var simplyHoistableInsts: [Instruction] = [] + var speculativelyHoistable: [Instruction] = [] var loadsAndStores: [Instruction] = [] var hoistUp: [Instruction] = [] var sinkDown: [Instruction] = [] @@ -66,8 +69,8 @@ private struct AnalyzedInstructions { } var globalInitCalls: Stack - var readOnlyApplies: StackWithCount - var loads: StackWithCount + var readOnlyApplies: Stack + var loads: Stack var stores: Stack var beginAccesses: Stack var fullApplies: Stack @@ -81,8 +84,8 @@ private struct AnalyzedInstructions { self.blockSideEffectBottomMarker = loopSideEffects.top self.globalInitCalls = Stack(context) - self.readOnlyApplies = StackWithCount(context) - self.loads = StackWithCount(context) + self.readOnlyApplies = Stack(context) + self.loads = Stack(context) self.stores = Stack(context) self.beginAccesses = Stack(context) self.fullApplies = Stack(context) @@ -103,17 +106,37 @@ private struct AnalyzedInstructions { } } -private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext) -> MovableInstructions? { - guard let preheader = loop.preheader else { - return nil - } - +/// Analyzes `loop` for hosting/sinking potential. +/// Computes `MovableInstructions` we may be able to move out of the loop +/// +/// This may split some loads into smaller loads. +private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext) -> MovableInstructions { + // TODO: Remove once uses lowered OSSA. loop.splitCriticalEdges(context) var movableInstructions = MovableInstructions() var analyzedInstructions = AnalyzedInstructions(context) defer { analyzedInstructions.deinitialize() } + analyzeInstructions(in: loop, &analyzedInstructions, &movableInstructions, context) + + collectHoistableReadOnlyApplies(analyzedInstructions, &movableInstructions, context) + + collectHoistableGlobalInitCalls(in: loop, analyzedInstructions, &movableInstructions, context) + + collectProjectableAccessPathsAndSplitLoads(in: loop, &analyzedInstructions, &movableInstructions, context) + + collectMovableInstructions(in: loop, &analyzedInstructions, &movableInstructions, context) + + return movableInstructions +} + +private func analyzeInstructions( + in loop: Loop, + _ analyzedInstructions: inout AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { for bb in loop.loopBlocks { analyzedInstructions.markBeginOfBlock() @@ -128,7 +151,7 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext continue } - + switch inst { case is FixLifetimeInst: break @@ -147,21 +170,21 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext analyzedInstructions.beginAccesses.append(beginAccessInst) analyzedInstructions.analyzeSideEffects(ofInst: beginAccessInst) case let refElementAddrInst as RefElementAddrInst: - movableInstructions.simplyHoistableInsts.append(refElementAddrInst) + movableInstructions.speculativelyHoistable.append(refElementAddrInst) case let condFailInst as CondFailInst: analyzedInstructions.analyzeSideEffects(ofInst: condFailInst) case let applyInst as ApplyInst: if applyInst.isSafeReadOnlyApply(context.calleeAnalysis) { analyzedInstructions.readOnlyApplies.append(applyInst) } else if let callee = applyInst.referencedFunction, - callee.isGlobalInitFunction, - !applyInst.globalInitMayConflictWith( - blockSideEffectSegment: analyzedInstructions.sideEffectsOfCurrentBlock, - context.aliasAnalysis - ) { + callee.isGlobalInitFunction, + !applyInst.globalInitMayConflictWith( + blockSideEffectSegment: analyzedInstructions.sideEffectsOfCurrentBlock, + context.aliasAnalysis + ) { analyzedInstructions.globalInitCalls.append(applyInst) } - + fallthrough default: switch inst { @@ -180,40 +203,60 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext } default: break } - + analyzedInstructions.analyzeSideEffects(ofInst: inst) - + if inst.canBeHoisted(outOf: loop, context) { movableInstructions.hoistUp.append(inst) } } } } +} - // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. - if analyzedInstructions.readOnlyApplies.count * analyzedInstructions.loopSideEffects.count < 8000 { - for readOnlyApply in analyzedInstructions.readOnlyApplies { - if readOnlyApply.isSafeReadOnlyApply( - for: analyzedInstructions.loopSideEffects, - context.aliasAnalysis, - context.calleeAnalysis - ) { - movableInstructions.hoistUp.append(readOnlyApply) - } +private func collectHoistableReadOnlyApplies( + _ analyzedInstructions: AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + var counter = 0 + for readOnlyApply in analyzedInstructions.readOnlyApplies { + // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. + if counter * analyzedInstructions.loopSideEffects.count < 8000, readOnlyApply.isSafeReadOnlyApply( + for: analyzedInstructions.loopSideEffects, + context.aliasAnalysis, + context.calleeAnalysis + ) { + movableInstructions.hoistUp.append(readOnlyApply) } + counter += 1 } +} +private func collectHoistableGlobalInitCalls( + in loop: Loop, + _ analyzedInstructions: AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { for globalInitCall in analyzedInstructions.globalInitCalls { if !globalInitCall.globalInitMayConflictWith( loopSideEffects: analyzedInstructions.loopSideEffects, - notPostDominatingPreheader: preheader, + notPostDominatingPreheader: loop.preheader!, context.aliasAnalysis, context.postDominatorTree ) { movableInstructions.hoistUp.append(globalInitCall) } } +} +private func collectProjectableAccessPathsAndSplitLoads( + in loop: Loop, + _ analyzedInstructions: inout AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { if !analyzedInstructions.hasOtherMemReadingInsts { for storeInst in analyzedInstructions.stores { let accessPath = storeInst.destination.accessPath @@ -226,15 +269,23 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext !movableInstructions.loadAndStoreAddrs.contains(accessPath), loop.storesCommonlyDominateExits(analyzedInstructions: analyzedInstructions, accessPath: accessPath, context), analyzedInstructions.splitLoads( - storeAddr: storeInst.destination, - accessPath: accessPath, - context + storeAddr: storeInst.destination, + accessPath: accessPath, + context ) { movableInstructions.loadAndStoreAddrs.append(accessPath) } } } - +} + +private func collectMovableInstructions( + in loop: Loop, + _ analyzedInstructions: inout AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + var loadInstCounter = 0 for bb in loop.loopBlocks { for inst in bb.instructions { // TODO: Remove once support for values with ownership implemented. @@ -244,7 +295,7 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext switch inst { case let fixLifetimeInst as FixLifetimeInst: - guard fixLifetimeInst.parentBlock.dominates(preheader, context.dominatorTree) else { + guard fixLifetimeInst.parentBlock.dominates(loop.preheader!, context.dominatorTree) else { continue } @@ -256,11 +307,13 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext } case let loadInst as LoadInst: // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. - if analyzedInstructions.loads.count * analyzedInstructions.loopSideEffects.count < 8000, + if loadInstCounter * analyzedInstructions.loopSideEffects.count < 8000, !loadInst.mayWriteTo(sideEffects: analyzedInstructions.loopSideEffects, context.aliasAnalysis) { movableInstructions.hoistUp.append(loadInst) } + loadInstCounter += 1 + movableInstructions.loadsAndStores.append(loadInst) case let storeInst as StoreInst: switch storeInst.storeOwnership { @@ -281,8 +334,6 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext } } } - - return movableInstructions } private func optimizeLoop( @@ -293,11 +344,11 @@ private func optimizeLoop( var changed = false // TODO: If we hoist tuple_element_addr and struct_element_addr instructions here, hoistAndSinkLoadsAndStores could converge after just one execution! - changed = movableInstructions.simpleHoistInstructions(loop: loop, context) || changed - changed = movableInstructions.hoistAndSinkLoadsAndStores(outOf: loop, context) || changed - changed = movableInstructions.hoistInstructions(loop: loop, context) || changed - changed = movableInstructions.sinkInstructions(loop: loop, context) || changed - changed = movableInstructions.hoistWithSinkScopedInstructions(loop: loop, context) || changed + changed = movableInstructions.speculativelyHoistInstructions(outOf: loop, context) || changed + changed = movableInstructions.hoistAndSinkLoadsAndStores(outOf: loop, context) || changed + changed = movableInstructions.hoistInstructions(outOf: loop, context) || changed + changed = movableInstructions.sinkInstructions(outOf: loop, context) || changed + changed = movableInstructions.hoistWithSinkScopedInstructions(outOf: loop, context) || changed return changed } @@ -345,9 +396,9 @@ private extension AnalyzedInstructions { } private extension MovableInstructions { - mutating func simpleHoistInstructions(loop: Loop, _ context: FunctionPassContext) -> Bool { + mutating func speculativelyHoistInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { var changed = false - for inst in simplyHoistableInsts { + for inst in speculativelyHoistable { changed = inst.hoist(outOf: loop, context) || changed } return changed @@ -362,10 +413,12 @@ private extension MovableInstructions { return changed } - mutating func hoistInstructions(loop: Loop, _ context: FunctionPassContext) -> Bool { + mutating func hoistInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(domTree: context.dominatorTree) var changed = false + // Only traverse instructions in blocks that dominate all exit and latch blocks. + // We don't hoist instructions speculatively here. for bb in dominatingBlocks { for inst in bb.instructions where hoistUp.contains(inst) { changed = inst.hoist(outOf: loop, context) || changed @@ -375,7 +428,7 @@ private extension MovableInstructions { return changed } - mutating func sinkInstructions(loop: Loop, _ context: FunctionPassContext) -> Bool { + mutating func sinkInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(domTree: context.dominatorTree) var changed = false @@ -386,7 +439,7 @@ private extension MovableInstructions { return changed } - mutating func hoistWithSinkScopedInstructions(loop: Loop, _ context: FunctionPassContext) -> Bool { + mutating func hoistWithSinkScopedInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { guard !loop.hasNoExitBlocks else { return false } @@ -417,14 +470,8 @@ private extension MovableInstructions { accessPath: AccessPath, context: FunctionPassContext ) -> Bool { - let exitingAndLatchBlocks = loop.exitingAndLatchBlocks - - guard let preheader = loop.preheader else { - return false - } - var changed = false - let builder = Builder(before: preheader.terminator, context) + let builder = Builder(before: loop.preheader!.terminator, context) var ssaUpdater: SSAUpdater? var storeAddr: Value? @@ -453,19 +500,19 @@ private extension MovableInstructions { return changed } - var cloner = Cloner(cloneBefore: preheader.terminator, context) + var cloner = Cloner(cloneBefore: loop.preheader!.terminator, context) defer { cloner.deinitialize() } guard let initialAddr = (cloner.cloneUseDefChain(addr: storeAddr) { srcAddr in - srcAddr.parentBlock.dominates(preheader, context.dominatorTree) + srcAddr.parentBlock.dominates(loop.preheader!, context.dominatorTree) }) else { return changed } - let ownership: LoadInst.LoadOwnership = preheader.terminator.parentFunction.hasOwnership ? .trivial : .unqualified + let ownership: LoadInst.LoadOwnership = loop.preheader!.terminator.parentFunction.hasOwnership ? .trivial : .unqualified let initialLoad = builder.createLoad(fromAddress: initialAddr, ownership: ownership) - ssaUpdater.addAvailableValue(initialLoad, in: preheader) + ssaUpdater.addAvailableValue(initialLoad, in: loop.preheader!) var currentBlock: BasicBlock? var currentVal: Value? @@ -513,7 +560,7 @@ private extension MovableInstructions { loadsAndStores.removeAll(where: { $0.isDeleted }) - for exitingOrLatchBlock in exitingAndLatchBlocks { + for exitingOrLatchBlock in loop.exitingAndLatchBlocks { for succesor in exitingOrLatchBlock.successors where !loop.contains(block: succesor) { guard let firstInst = succesor.instructions.first else { continue @@ -540,6 +587,7 @@ private extension MovableInstructions { } private extension Loop { + /// Returns `true` if all stores to `accessPath` commonly dominate the loop exits. func storesCommonlyDominateExits( analyzedInstructions: AnalyzedInstructions, accessPath: AccessPath, @@ -591,18 +639,15 @@ private extension Instruction { || operands.contains(where: { $0.value.ownership != .none }) } + /// Returns `true` if this instruction follows the default hoisting heuristic. func canBeHoisted(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { - guard let preheader = loop.preheader else { - return false - } - switch self { case is TermInst, is Allocation, is Deallocation: return false case is ApplyInst: switch arraySemanticsCallKind { case .getCount, .getCapacity: - if canHoistArraySemanticsCall(to: preheader.terminator, context) { + if canHoistArraySemanticsCall(to: loop.preheader!.terminator, context) { return true } case .arrayPropsIsNativeTypeChecked: @@ -622,12 +667,11 @@ private extension Instruction { } func hoist(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { - guard let preheader = loop.preheader, - operands.allSatisfy({ !loop.contains(block: $0.value.parentBlock) }) else { + guard operands.allSatisfy({ !loop.contains(block: $0.value.parentBlock) }) else { return false } - let terminator = preheader.terminator + let terminator = loop.preheader!.terminator if canHoistArraySemanticsCall(to: terminator, context) { hoistArraySemanticsCall(before: terminator, context) } else { @@ -635,8 +679,8 @@ private extension Instruction { } if let singleValueInst = self as? SingleValueInstruction, - !(self is BeginAccessInst), - let identicalInst = (preheader.instructions.first { otherInst in + !(self is BeginAccessInst || self is IntegerLiteralInst), + let identicalInst = (loop.preheader!.instructions.first { otherInst in return singleValueInst != otherInst && singleValueInst.isIdenticalTo(otherInst) }) { guard let identicalSingleValueInst = identicalInst as? SingleValueInstruction else { @@ -683,6 +727,8 @@ private extension Instruction { return changed } + /// Returns `true` if `sideEffect` cannot be reordered with a call to this + /// global initializer. private func globalInitMayConflictWith( sideEffect: Instruction, _ aliasAnalysis: AliasAnalysis @@ -699,6 +745,9 @@ private extension Instruction { } } + /// Returns `true` if any of the instructions in `sideEffects` cannot be + /// reordered with a call to this global initializer (which is in the same basic + /// block). func globalInitMayConflictWith( blockSideEffectSegment: StackWithCount.Segment, _ aliasAnalysis: AliasAnalysis @@ -712,6 +761,9 @@ private extension Instruction { } } + /// Returns `true` if any of the instructions in `loopSideEffects` which are + /// post-dominated by a call to this global initializer cannot be reordered with + /// the call. func globalInitMayConflictWith( loopSideEffects: StackWithCount, notPostDominatingPreheader: BasicBlock, @@ -724,6 +776,10 @@ private extension Instruction { return loopSideEffects .contains { sideEffect in + // Only check instructions in blocks which are "before" (i.e. post-dominated + // by) the block which contains the init-call. + // Instructions which are before the call in the same block have already + // been checked. parentBlock.strictlyPostDominates( sideEffect.parentBlock, postDomTree @@ -743,6 +799,8 @@ private extension Instruction { } private extension UnaryInstruction { + /// Returns true if `sideEffects` contains any memory writes which + /// may alias with the memory addressed by this instruction. func mayWriteTo( sideEffects: StackWithCount, _ aliasAnalysis: AliasAnalysis @@ -755,6 +813,7 @@ private extension UnaryInstruction { } private extension StoreInst { + /// Returns a `true` if this store is a store to `accessPath`. func storesTo(_ accessPath: AccessPath) -> Bool { guard self.storeOwnership != .initialize else { return false @@ -765,6 +824,8 @@ private extension StoreInst { } private extension LoadInst { + /// Returns `true` if this load instruction loads from `accessPath` or a + /// projected address from `accessPath`. func loadsFrom(_ accessPath: AccessPath) -> Bool { return accessPath.getProjection(to: self.address.accessPath)?.isMaterializable ?? false } @@ -911,6 +972,11 @@ private extension AccessPath { return projectionPath.isConstant } + /// Returns true if all instructions in `sideEffects` which may alias with + /// this path are either loads or stores from this path. + /// + /// `storeAddr` is only needed for AliasAnalysis until we have an interface + /// that supports `AccessPath`. func isOnlyLoadedAndStored( storeAddr: Value, analyzedInstructions: AnalyzedInstructions, diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift index 594514795672f..4e265cf0ac64a 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift @@ -154,7 +154,7 @@ extension MutatingContext { } extension Instruction { - var arraySemanticsCallKind: BridgedArrayCallKind { + var arraySemanticsCallKind: ArrayCallKind { return BridgedPassContext.getArraySemanticsCallKind(self.bridged) } diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift index dd739bdfe5b96..6679735fbf3d1 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift @@ -30,25 +30,25 @@ struct Cloner { let target: Target init(cloneToGlobal: GlobalVariable, _ context: Context) { - self.bridged = BridgedCloner(cloneToGlobal.bridged, context.bridgedPassContext) + self.bridged = BridgedCloner(cloneToGlobal.bridged, context._bridged) self.context = context self.target = .global(cloneToGlobal) } init(cloneBefore inst: Instruction, _ context: Context) { - self.bridged = BridgedCloner(inst.bridged, context.bridgedPassContext) + self.bridged = BridgedCloner(inst.bridged, context._bridged) self.context = context self.target = .function(inst.parentFunction) } init(cloneToEmptyFunction: Function, _ context: Context) where Context == FunctionPassContext { - self.bridged = BridgedCloner(cloneToEmptyFunction.bridged, context.bridgedPassContext) + self.bridged = BridgedCloner(cloneToEmptyFunction.bridged, context._bridged) self.context = context self.target = .function(cloneToEmptyFunction) } mutating func deinitialize() { - bridged.destroy(context.bridgedPassContext) + bridged.destroy(context._bridged) } var targetFunction: Function { @@ -98,12 +98,12 @@ struct Cloner { } mutating func cloneUseDefChain(addr: Value, checkBase: (Value) -> Bool) -> Value? { - // MARK: Hacky temp fix + // TODO: Temp fix if addr is AllocStackInst { return nil } - guard !checkBase(addr) else { + if checkBase(addr) { bridged.recordFoldedValue(addr.bridged, addr.bridged) return addr } @@ -113,7 +113,6 @@ struct Cloner { // end_borrows or fix mark_dependence operands. case is BeginBorrowInst, is MarkDependenceInst: return nil case let singleValueInstruction as SingleValueInstruction: - // TODO: Double check whether correct guard let sourceOperand = singleValueInstruction.operands.first else { return nil } return cloneProjection(projectAddr: singleValueInstruction, sourceOperand: sourceOperand, checkBase: checkBase) diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift index 443959301770f..83c5cdbb762f0 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift @@ -600,37 +600,21 @@ extension StoreInst { extension LoadInst { @discardableResult func trySplit(_ context: FunctionPassContext) -> Bool { - var elements = [LoadInst]() - let builder = Builder(before: self, context) - if type.isStruct { - if (type.nominal as! StructDecl).hasUnreferenceableStorage { - return false - } + if type.isStruct && (type.nominal as! StructDecl).hasUnreferenceableStorage { guard let fields = type.getNominalFields(in: parentFunction) else { return false } - for idx in 0.. LoadOwnership { @@ -646,54 +630,63 @@ extension LoadInst { alongPath projectionPath: SmallProjectionPath, _ context: FunctionPassContext ) -> [LoadInst]? { - guard !projectionPath.isEmpty else { + if projectionPath.isEmpty { return nil } let (fieldKind, index, pathRemainder) = projectionPath.pop() - var elements = [LoadInst]() - let builder = Builder(before: self, context) + var elements: [LoadInst] switch fieldKind { case .structField where !(type.nominal as! StructDecl).hasUnreferenceableStorage && type.isStruct: guard let fields = type.getNominalFields(in: parentFunction) else { return nil } - for idx in 0.. [LoadInst] { + var elements = [LoadInst]() + let builder = Builder(before: self, context) + + for idx in 0.. [LoadInst] { + var elements = [LoadInst]() + let builder = Builder(before: self, context) + + for idx in 0..(semCall.getKind()); + return semCall.getKind(); } bool BridgedPassContext::canHoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction toInst) const { diff --git a/lib/SIL/Utils/SILBridging.cpp b/lib/SIL/Utils/SILBridging.cpp index 546edf0138ad5..4b289aa0fee76 100644 --- a/lib/SIL/Utils/SILBridging.cpp +++ b/lib/SIL/Utils/SILBridging.cpp @@ -545,6 +545,146 @@ BridgedInstruction BridgedBuilder::createSwitchEnumAddrInst(BridgedValue enumAdd convertCases(enumAddr.getSILValue()->getType(), enumCases, numEnumCases))}; } +//===----------------------------------------------------------------------===// +// BridgedCloner +//===----------------------------------------------------------------------===// + +// Need to put ClonerWithFixedLocation into namespace swift to forward reference +// it in OptimizerBridging.h. +namespace swift { + +class BridgedClonerImpl : public SILCloner { + friend class SILInstructionVisitor; + friend class SILCloner; + + bool hasFixedLocation; + union { + SILDebugLocation fixedLocation; + ScopeCloner scopeCloner; + }; + + SILInstruction *result = nullptr; + +public: + BridgedClonerImpl(SILGlobalVariable *gVar) + : SILCloner(gVar), + hasFixedLocation(true), + fixedLocation(ArtificialUnreachableLocation(), nullptr) {} + + BridgedClonerImpl(SILInstruction *insertionPoint) + : SILCloner(*insertionPoint->getFunction()), + hasFixedLocation(true), + fixedLocation(insertionPoint->getDebugLocation()) { + Builder.setInsertionPoint(insertionPoint); + } + + BridgedClonerImpl(SILFunction &emptyFunction) + : SILCloner(emptyFunction), + hasFixedLocation(false), + scopeCloner(ScopeCloner(emptyFunction)) {} + + ~BridgedClonerImpl() { + if (hasFixedLocation) { + fixedLocation.~SILDebugLocation(); + } else { + scopeCloner.~ScopeCloner(); + } + } + + SILValue getClonedValue(SILValue v) { + return getMappedValue(v); + } + + SILInstruction *cloneInst(SILInstruction *inst) { + result = nullptr; + visit(inst); + ASSERT(result && "instruction not cloned"); + return result; + } + + SILLocation remapLocation(SILLocation loc) { + if (hasFixedLocation) + return fixedLocation.getLocation(); + return loc; + } + + const SILDebugScope *remapScope(const SILDebugScope *DS) { + if (hasFixedLocation) + return fixedLocation.getScope(); + return scopeCloner.getOrCreateClonedScope(DS); + } + + void postProcess(SILInstruction *Orig, SILInstruction *Cloned) { + result = Cloned; + SILCloner::postProcess(Orig, Cloned); + } + +}; + +} // namespace swift + +BridgedCloner::BridgedCloner(BridgedGlobalVar var, BridgedContext context) + : cloner(new BridgedClonerImpl(var.getGlobal())) { + context.context->notifyNewCloner(); +} + +BridgedCloner::BridgedCloner(BridgedInstruction inst, + BridgedContext context) + : cloner(new BridgedClonerImpl(inst.unbridged())) { + context.context->notifyNewCloner(); +} + +BridgedCloner::BridgedCloner(BridgedFunction emptyFunction, BridgedContext context) + : cloner(new BridgedClonerImpl(*emptyFunction.getFunction())) { + context.context->notifyNewCloner(); +} + +void BridgedCloner::destroy(BridgedContext context) { + delete cloner; + cloner = nullptr; + context.context->notifyClonerDestroyed(); +} + +BridgedFunction BridgedCloner::getCloned() const { + return { &cloner->getBuilder().getFunction() }; +} + +BridgedValue BridgedCloner::getClonedValue(BridgedValue v) { + return {cloner->getClonedValue(v.getSILValue())}; +} + +bool BridgedCloner::isValueCloned(BridgedValue v) const { + return cloner->isValueCloned(v.getSILValue()); +} + +void BridgedCloner::recordClonedInstruction(BridgedInstruction origInst, BridgedInstruction clonedInst) const { + cloner->recordClonedInstruction(origInst.unbridged(), clonedInst.unbridged()); +} + +void BridgedCloner::recordFoldedValue(BridgedValue orig, BridgedValue mapped) const { + cloner->recordFoldedValue(orig.getSILValue(), mapped.getSILValue()); +} + +BridgedInstruction BridgedCloner::clone(BridgedInstruction inst) { + return {cloner->cloneInst(inst.unbridged())->asSILNode()}; +} + +BridgedBasicBlock BridgedCloner::getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const { + return { cloner->getOpBasicBlock(originalBasicBlock.unbridged()) }; +} + +void BridgedCloner::cloneFunctionBody(BridgedFunction originalFunction, + BridgedBasicBlock clonedEntryBlock, + BridgedValueArray clonedEntryBlockArgs) const { + llvm::SmallVector clonedEntryBlockArgsStorage; + auto clonedEntryBlockArgsArrayRef = clonedEntryBlockArgs.getValues(clonedEntryBlockArgsStorage); + cloner->cloneFunctionBody(originalFunction.getFunction(), clonedEntryBlock.unbridged(), clonedEntryBlockArgsArrayRef); +} + +void BridgedCloner::cloneFunctionBody(BridgedFunction originalFunction) const { + cloner->cloneFunction(originalFunction.getFunction()); +} + //===----------------------------------------------------------------------===// // BridgedContext //===----------------------------------------------------------------------===// diff --git a/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp b/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp index 0b12222f412b0..b9a1314a2fd2e 100644 --- a/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp +++ b/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp @@ -337,7 +337,7 @@ void ArrayInfo::classifyUsesOfArray(SILValue arrayValue) { if (arrayOp.doesNotChangeArray()) continue; - if (arrayOp.getKind() == swift::ArrayCallKind::kArrayFinalizeIntrinsic) { + if (arrayOp.getKind() == ArrayCallKind::kArrayFinalizeIntrinsic) { classifyUsesOfArray((ApplyInst *)arrayOp); continue; } diff --git a/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp b/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp index 6766f7b5ae5a4..34cbe692152fe 100644 --- a/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp +++ b/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp @@ -685,7 +685,7 @@ recursivelyCollectInteriorUses(ValueBase *DefInst, continue; } ArraySemanticsCall AS(svi); - if (AS.getKind() == swift::ArrayCallKind::kArrayFinalizeIntrinsic) { + if (AS.getKind() == ArrayCallKind::kArrayFinalizeIntrinsic) { if (!recursivelyCollectInteriorUses(svi, AddressNode, IsInteriorAddress)) return false; continue; diff --git a/lib/SILOptimizer/Utils/OptimizerBridging.cpp b/lib/SILOptimizer/Utils/OptimizerBridging.cpp index 65e10c3f7f5da..073f012be8c67 100644 --- a/lib/SILOptimizer/Utils/OptimizerBridging.cpp +++ b/lib/SILOptimizer/Utils/OptimizerBridging.cpp @@ -457,142 +457,6 @@ BridgedCalleeAnalysis::CalleeList BridgedCalleeAnalysis::getDestructors(BridgedT return ca->getDestructors(type.unbridged(), isExactType); } -// Need to put ClonerWithFixedLocation into namespace swift to forward reference -// it in OptimizerBridging.h. -namespace swift { - -class BridgedClonerImpl : public SILCloner { - friend class SILInstructionVisitor; - friend class SILCloner; - - bool hasFixedLocation; - union { - SILDebugLocation fixedLocation; - ScopeCloner scopeCloner; - }; - - SILInstruction *result = nullptr; - -public: - BridgedClonerImpl(SILGlobalVariable *gVar) - : SILCloner(gVar), - hasFixedLocation(true), - fixedLocation(ArtificialUnreachableLocation(), nullptr) {} - - BridgedClonerImpl(SILInstruction *insertionPoint) - : SILCloner(*insertionPoint->getFunction()), - hasFixedLocation(true), - fixedLocation(insertionPoint->getDebugLocation()) { - Builder.setInsertionPoint(insertionPoint); - } - - BridgedClonerImpl(SILFunction &emptyFunction) - : SILCloner(emptyFunction), - hasFixedLocation(false), - scopeCloner(ScopeCloner(emptyFunction)) {} - - ~BridgedClonerImpl() { - if (hasFixedLocation) { - fixedLocation.~SILDebugLocation(); - } else { - scopeCloner.~ScopeCloner(); - } - } - - SILValue getClonedValue(SILValue v) { - return getMappedValue(v); - } - - SILInstruction *cloneInst(SILInstruction *inst) { - result = nullptr; - visit(inst); - ASSERT(result && "instruction not cloned"); - return result; - } - - SILLocation remapLocation(SILLocation loc) { - if (hasFixedLocation) - return fixedLocation.getLocation(); - return loc; - } - - const SILDebugScope *remapScope(const SILDebugScope *DS) { - if (hasFixedLocation) - return fixedLocation.getScope(); - return scopeCloner.getOrCreateClonedScope(DS); - } - - void postProcess(SILInstruction *Orig, SILInstruction *Cloned) { - result = Cloned; - SILCloner::postProcess(Orig, Cloned); - } - -}; - -} // namespace swift - -BridgedCloner::BridgedCloner(BridgedGlobalVar var, BridgedPassContext context) - : cloner(new BridgedClonerImpl(var.getGlobal())) { - context.invocation->notifyNewCloner(); -} - -BridgedCloner::BridgedCloner(BridgedInstruction inst, - BridgedPassContext context) - : cloner(new BridgedClonerImpl(inst.unbridged())) { - context.invocation->notifyNewCloner(); -} - -BridgedCloner::BridgedCloner(BridgedFunction emptyFunction, BridgedPassContext context) - : cloner(new BridgedClonerImpl(*emptyFunction.getFunction())) { - context.invocation->notifyNewCloner(); -} - -void BridgedCloner::destroy(BridgedPassContext context) { - delete cloner; - cloner = nullptr; - context.invocation->notifyClonerDestroyed(); -} - -BridgedFunction BridgedCloner::getCloned() const { - return { &cloner->getBuilder().getFunction() }; -} - -BridgedValue BridgedCloner::getClonedValue(BridgedValue v) { - return {cloner->getClonedValue(v.getSILValue())}; -} - -bool BridgedCloner::isValueCloned(BridgedValue v) const { - return cloner->isValueCloned(v.getSILValue()); -} - -void BridgedCloner::recordClonedInstruction(BridgedInstruction origInst, BridgedInstruction clonedInst) const { - cloner->recordClonedInstruction(origInst.unbridged(), clonedInst.unbridged()); -} - -void BridgedCloner::recordFoldedValue(BridgedValue orig, BridgedValue mapped) const { - cloner->recordFoldedValue(orig.getSILValue(), mapped.getSILValue()); -} - -BridgedInstruction BridgedCloner::clone(BridgedInstruction inst) { - return {cloner->cloneInst(inst.unbridged())->asSILNode()}; -} - -BridgedBasicBlock BridgedCloner::getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const { - return { cloner->getOpBasicBlock(originalBasicBlock.unbridged()) }; -} - -void BridgedCloner::cloneFunctionBody(BridgedFunction originalFunction, - BridgedBasicBlock clonedEntryBlock, - BridgedValueArray clonedEntryBlockArgs) const { - llvm::SmallVector clonedEntryBlockArgsStorage; - auto clonedEntryBlockArgsArrayRef = clonedEntryBlockArgs.getValues(clonedEntryBlockArgsStorage); - cloner->cloneFunctionBody(originalFunction.getFunction(), clonedEntryBlock.unbridged(), clonedEntryBlockArgsArrayRef); -} - -void BridgedCloner::cloneFunctionBody(BridgedFunction originalFunction) const { - cloner->cloneFunction(originalFunction.getFunction()); -} - void BridgedBuilder::destroyCapturedArgs(BridgedInstruction partialApply) const { if (auto *pai = llvm::dyn_cast(partialApply.unbridged()); pai->isOnStack()) { auto b = unbridged(); diff --git a/test/SILOptimizer/licm_exclusivity.swift b/test/SILOptimizer/licm_exclusivity.swift index 458a57c42e0d8..0b9f56e6bd889 100644 --- a/test/SILOptimizer/licm_exclusivity.swift +++ b/test/SILOptimizer/licm_exclusivity.swift @@ -1,17 +1,9 @@ -// RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -Xllvm -debug-only=sil-licm -primary-file %s 2>&1 | %FileCheck %s --check-prefix=TESTLICM -// RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -Xllvm -debug-only=sil-licm -primary-file %s 2>&1 | %FileCheck %s --check-prefix=TESTLICM2 // RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -primary-file %s | %FileCheck %s --check-prefix=TESTSIL -// RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -Xllvm -debug-only=sil-licm -whole-module-optimization %s 2>&1 | %FileCheck %s --check-prefix=TESTLICMWMO // RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -whole-module-optimization %s | %FileCheck %s --check-prefix=TESTSILWMO // REQUIRES: optimized_stdlib,asserts,swift_stdlib_no_asserts // REQUIRES: PTRSIZE=64 -// TESTLICM-LABEL: Processing loops in {{.*}}run_ReversedArray{{.*}} -// TESTLICM: Hoist and Sink pairs attempt -// TESTLICM: Hoisted -// TESTLICM: Successfully hoisted and sank pair - // TESTSIL-LABEL: sil hidden @$s16licm_exclusivity17run_ReversedArrayyySiF : $@convention(thin) (Int) -> () { // TESTSIL: bb // TESTSIL: begin_access [modify] [dynamic] [no_nested_conflict] @@ -35,10 +27,6 @@ func run_ReversedArray(_ N: Int) { } } -// TESTLICM2-LABEL: Processing loops in {{.*}}count_unicodeScalars{{.*}} -// TESTLICM2: Hoist and Sink pairs attempt -// TESTLICM2: Hoisted - // TESTSIL-LABEL: sil @$s16licm_exclusivity20count_unicodeScalarsyySS17UnicodeScalarViewVF : $@convention(thin) (@guaranteed String.UnicodeScalarView) -> () { // TESTSIL: bb0(%0 : $String.UnicodeScalarView) // TESTSIL: bb5: @@ -66,12 +54,6 @@ public class ClassWithArrs { B = [Int](repeating: 0, count: N) } -// TESTLICMWMO-LABEL: Processing loops in {{.*}}ClassWithArrsC7readArr{{.*}} -// TESTLICMWMO: Hoist and Sink pairs attempt -// TESTLICMWMO: Hoisted -// TESTLICMWMO: Successfully hoisted and sank pair -// TESTLICMWMO: Hoisted -// TESTLICMWMO: Successfully hoisted and sank pair // TESTSILWMO-LABEL: sil {{.*}}@$s16licm_exclusivity13ClassWithArrsC7readArryyF : $@convention(method) (@guaranteed ClassWithArrs) -> () { // TESTSILWMO: [[R1:%.*]] = ref_element_addr %0 : $ClassWithArrs, #ClassWithArrs.A // TESTSILWMO: [[R2:%.*]] = ref_element_addr %0 : $ClassWithArrs, #ClassWithArrs.B From af189a4065f725004853de63943dd46d7afe2d74 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Mon, 18 Aug 2025 12:04:37 +0100 Subject: [PATCH 36/44] Move cloner code to SIL module. Fix minor bug in trySplit. --- .../LoopInvariantCodeMotion.swift | 1 + .../Optimizer/PassManager/ContextCommon.swift | 26 ++++++++ .../Optimizer/Utilities/CMakeLists.txt | 1 - .../Optimizer/Utilities/OptUtils.swift | 10 +-- .../Sources/SIL/Utilities/CMakeLists.txt | 1 + .../{Optimizer => SIL}/Utilities/Cloner.swift | 62 +++++-------------- 6 files changed, 51 insertions(+), 50 deletions(-) rename SwiftCompilerSources/Sources/{Optimizer => SIL}/Utilities/Cloner.swift (69%) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 9ed4ef72dc158..ef6cbe07bb9b3 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -678,6 +678,7 @@ private extension Instruction { move(before: terminator, context) } + // TODO: `self is IntegerLiteralInst` is required due to a possible bug in bounds check opt. if let singleValueInst = self as? SingleValueInstruction, !(self is BeginAccessInst || self is IntegerLiteralInst), let identicalInst = (loop.preheader!.instructions.first { otherInst in diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift index 4e265cf0ac64a..eab71e27166cb 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/ContextCommon.swift @@ -167,3 +167,29 @@ extension Instruction { context.notifyInstructionsChanged() } } + +extension Cloner where Context == FunctionPassContext { + func getOrCreateEntryBlock() -> BasicBlock { + if let entryBlock = targetFunction.blocks.first { + return entryBlock + } + return targetFunction.appendNewBlock(context) + } + + func cloneFunctionBody(from originalFunction: Function, entryBlockArguments: [Value]) { + entryBlockArguments.withBridgedValues { bridgedEntryBlockArgs in + let entryBlock = getOrCreateEntryBlock() + bridged.cloneFunctionBody(originalFunction.bridged, entryBlock.bridged, bridgedEntryBlockArgs) + } + } + + func cloneFunctionBody(from originalFunction: Function) { + bridged.cloneFunctionBody(originalFunction.bridged) + } +} + +func cloneFunction(from originalFunction: Function, toEmpty targetFunction: Function, _ context: FunctionPassContext) { + var cloner = Cloner(cloneToEmptyFunction: targetFunction, context) + defer { cloner.deinitialize() } + cloner.cloneFunctionBody(from: originalFunction) +} diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt index 55d6d3c6fbc1d..35ca4d97aae75 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt @@ -8,7 +8,6 @@ swift_compiler_sources(Optimizer AddressUtils.swift - Cloner.swift Devirtualization.swift EscapeUtils.swift FunctionSignatureTransforms.swift diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift index 83c5cdbb762f0..a65df88634cbf 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift @@ -600,8 +600,9 @@ extension StoreInst { extension LoadInst { @discardableResult func trySplit(_ context: FunctionPassContext) -> Bool { - if type.isStruct && (type.nominal as! StructDecl).hasUnreferenceableStorage { - guard let fields = type.getNominalFields(in: parentFunction) else { + if type.isStruct { + guard !(type.nominal as! StructDecl).hasUnreferenceableStorage, + let fields = type.getNominalFields(in: parentFunction) else { return false } @@ -639,8 +640,9 @@ extension LoadInst { var elements: [LoadInst] switch fieldKind { - case .structField where !(type.nominal as! StructDecl).hasUnreferenceableStorage && type.isStruct: - guard let fields = type.getNominalFields(in: parentFunction) else { + case .structField where type.isStruct: + guard !(type.nominal as! StructDecl).hasUnreferenceableStorage, + let fields = type.getNominalFields(in: parentFunction) else { return nil } diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt b/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt index f39a2a89ae0e3..59ba3af5f8912 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt @@ -9,6 +9,7 @@ swift_compiler_sources(SIL AccessUtils.swift BorrowUtils.swift + Cloner.swift ForwardingUtils.swift PhiUpdater.swift SequenceUtilities.swift diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift similarity index 69% rename from SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift rename to SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift index 6679735fbf3d1..8eb73304663ce 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/Cloner.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// import OptimizerBridging -import SIL import SILBridging /// Clones the initializer value of a GlobalVariable. @@ -19,46 +18,46 @@ import SILBridging /// Used to transitively clone "constant" instructions, including their operands, /// from or to the static initializer value of a GlobalVariable. /// -struct Cloner { - private var bridged: BridgedCloner - let context: Context +public struct Cloner { + public var bridged: BridgedCloner + public let context: Context - enum Target { + public enum Target { case function(Function) case global(GlobalVariable) } - let target: Target + public let target: Target - init(cloneToGlobal: GlobalVariable, _ context: Context) { + public init(cloneToGlobal: GlobalVariable, _ context: Context) { self.bridged = BridgedCloner(cloneToGlobal.bridged, context._bridged) self.context = context self.target = .global(cloneToGlobal) } - init(cloneBefore inst: Instruction, _ context: Context) { + public init(cloneBefore inst: Instruction, _ context: Context) { self.bridged = BridgedCloner(inst.bridged, context._bridged) self.context = context self.target = .function(inst.parentFunction) } - - init(cloneToEmptyFunction: Function, _ context: Context) where Context == FunctionPassContext { + + public init(cloneToEmptyFunction: Function, _ context: Context) { self.bridged = BridgedCloner(cloneToEmptyFunction.bridged, context._bridged) self.context = context self.target = .function(cloneToEmptyFunction) } - mutating func deinitialize() { + public mutating func deinitialize() { bridged.destroy(context._bridged) } - var targetFunction: Function { + public var targetFunction: Function { guard case .function(let function) = target else { fatalError("expected cloning into a function") } return function } - mutating func clone(instruction: Instruction) -> Instruction { + public mutating func clone(instruction: Instruction) -> Instruction { let cloned = bridged.clone(instruction.bridged).instruction if case .function = target { context.notifyInstructionChanged(cloned) @@ -68,7 +67,7 @@ struct Cloner { } /// Transitively clones `value` including its defining instruction's operands. - mutating func cloneRecursively(value: Value) -> Value { + public mutating func cloneRecursively(value: Value) -> Value { if isCloned(value: value) { return getClonedValue(of: value) } @@ -97,7 +96,7 @@ struct Cloner { fatalError("unexpected instruction kind") } - mutating func cloneUseDefChain(addr: Value, checkBase: (Value) -> Bool) -> Value? { + public mutating func cloneUseDefChain(addr: Value, checkBase: (Value) -> Bool) -> Value? { // TODO: Temp fix if addr is AllocStackInst { return nil @@ -137,42 +136,15 @@ struct Cloner { return clone as! SingleValueInstruction } - mutating func getClonedValue(of originalValue: Value) -> Value { + public mutating func getClonedValue(of originalValue: Value) -> Value { bridged.getClonedValue(originalValue.bridged).value } - func isCloned(value: Value) -> Bool { + public func isCloned(value: Value) -> Bool { bridged.isValueCloned(value.bridged) } - func getClonedBlock(for originalBlock: BasicBlock) -> BasicBlock { + public func getClonedBlock(for originalBlock: BasicBlock) -> BasicBlock { bridged.getClonedBasicBlock(originalBlock.bridged).block } - -} - -extension Cloner where Context == FunctionPassContext { - func getOrCreateEntryBlock() -> BasicBlock { - if let entryBlock = targetFunction.blocks.first { - return entryBlock - } - return targetFunction.appendNewBlock(context) - } - - func cloneFunctionBody(from originalFunction: Function, entryBlockArguments: [Value]) { - entryBlockArguments.withBridgedValues { bridgedEntryBlockArgs in - let entryBlock = getOrCreateEntryBlock() - bridged.cloneFunctionBody(originalFunction.bridged, entryBlock.bridged, bridgedEntryBlockArgs) - } - } - - func cloneFunctionBody(from originalFunction: Function) { - bridged.cloneFunctionBody(originalFunction.bridged) - } -} - -func cloneFunction(from originalFunction: Function, toEmpty targetFunction: Function, _ context: FunctionPassContext) { - var cloner = Cloner(cloneToEmptyFunction: targetFunction, context) - defer { cloner.deinitialize() } - cloner.cloneFunctionBody(from: originalFunction) } From acf36332e192a5a256283dfe4a2a71ea1796c47f Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Tue, 19 Aug 2025 11:51:02 +0100 Subject: [PATCH 37/44] Refactoring. Adjust inline array bounds check tests. --- .../Sources/Optimizer/Analysis/LoopTree.swift | 9 +++--- .../LoopInvariantCodeMotion.swift | 29 ++++++++++--------- .../Sources/SIL/Utilities/Cloner.swift | 4 +-- .../swift/SILOptimizer/OptimizerBridging.h | 2 +- .../SILOptimizer/OptimizerBridgingImpl.h | 2 +- .../inlinearray_bounds_check_tests.swift | 4 +-- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index 59bb3625a80fb..d961bafab565c 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -64,7 +64,8 @@ struct Loop { bridged.getHeader().block } - var isSingleExit: Bool { + /// Returns `true` if the loop has exactly one exit block. + var hasSingleExitBlock: Bool { return exitBlocks.singleElement != nil } @@ -181,10 +182,8 @@ func splitEdge( dominatorTree: DominatorTree, loopTree: LoopTree, _ context: some MutatingContext -) -> BasicBlock? { - guard let result = loopTree.bridged.splitEdge(block.bridged, toEdgeIndex, dominatorTree.bridged).block else { - return nil - } +) -> BasicBlock { + let result = loopTree.bridged.splitEdge(block.bridged, toEdgeIndex, dominatorTree.bridged).block context.notifyBranchesChanged() return result diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index ef6cbe07bb9b3..5b7abb51bc347 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -472,7 +472,6 @@ private extension MovableInstructions { ) -> Bool { var changed = false let builder = Builder(before: loop.preheader!.terminator, context) - var ssaUpdater: SSAUpdater? var storeAddr: Value? for case let storeInst as StoreInst in loadsAndStores where storeInst.storesTo(accessPath) { @@ -483,27 +482,30 @@ private extension MovableInstructions { if storeAddr == nil { storeAddr = storeInst.destination - ssaUpdater = SSAUpdater( - function: storeAddr!.parentFunction, - type: storeAddr!.type.objectType, - ownership: storeAddr!.ownership, - context - ) } else if storeInst.destination.type != storeAddr!.type { return changed } - - ssaUpdater?.addAvailableValue(storeInst.source, in: storeInst.parentBlock) } - guard let storeAddr, var ssaUpdater else { + guard let storeAddr else { return changed } + var ssaUpdater = SSAUpdater( + function: storeAddr.parentFunction, + type: storeAddr.type.objectType, + ownership: storeAddr.ownership, + context + ) + + for case let storeInst as StoreInst in loadsAndStores where storeInst.storesTo(accessPath) { + ssaUpdater.addAvailableValue(storeInst.source, in: storeInst.parentBlock) + } + var cloner = Cloner(cloneBefore: loop.preheader!.terminator, context) defer { cloner.deinitialize() } - guard let initialAddr = (cloner.cloneUseDefChain(addr: storeAddr) { srcAddr in + guard let initialAddr = (cloner.cloneAddressProjections(addr: storeAddr) { srcAddr in srcAddr.parentBlock.dominates(loop.preheader!, context.dominatorTree) }) else { return changed @@ -678,9 +680,8 @@ private extension Instruction { move(before: terminator, context) } - // TODO: `self is IntegerLiteralInst` is required due to a possible bug in bounds check opt. if let singleValueInst = self as? SingleValueInstruction, - !(self is BeginAccessInst || self is IntegerLiteralInst), + !(self is BeginAccessInst), let identicalInst = (loop.preheader!.instructions.first { otherInst in return singleValueInst != otherInst && singleValueInst.isIdenticalTo(otherInst) }) { @@ -695,7 +696,7 @@ private extension Instruction { } func sink(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { - var canMove = loop.isSingleExit + var canMove = loop.hasSingleExitBlock let exitBlocks = loop.exitBlocks let exitingBlocks = loop.exitingBlocks var newExitBlocks = Stack(context) diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift index 8eb73304663ce..f79afd8264c77 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift @@ -96,7 +96,7 @@ public struct Cloner { fatalError("unexpected instruction kind") } - public mutating func cloneUseDefChain(addr: Value, checkBase: (Value) -> Bool) -> Value? { + public mutating func cloneAddressProjections(addr: Value, checkBase: (Value) -> Bool) -> Value? { // TODO: Temp fix if addr is AllocStackInst { return nil @@ -124,7 +124,7 @@ public struct Cloner { sourceOperand: Operand, checkBase: (Value) -> Bool ) -> Value? { - guard let projectedSource = cloneUseDefChain( + guard let projectedSource = cloneAddressProjections( addr: sourceOperand.value, checkBase: checkBase ) else { diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index 5d9e909352703..e553c96cdb755 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -131,7 +131,7 @@ struct BridgedLoopTree { BRIDGED_INLINE SwiftInt getTopLevelLoopCount() const; BRIDGED_INLINE BridgedLoop getLoop(SwiftInt index) const; - SWIFT_IMPORT_UNSAFE BRIDGED_INLINE OptionalBridgedBasicBlock splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedBasicBlock splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const; }; struct BridgedPassContext { diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index 6b0014d34a747..f700122cd64cd 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -109,7 +109,7 @@ BridgedLoop BridgedLoopTree::getLoop(SwiftInt index) const { return {li->begin()[index]}; } -OptionalBridgedBasicBlock BridgedLoopTree::splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const { +BridgedBasicBlock BridgedLoopTree::splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const { return {swift::splitEdge(bb.unbridged()->getTerminator(), edgeIndex, domTree.di, li)}; } diff --git a/test/SILOptimizer/inlinearray_bounds_check_tests.swift b/test/SILOptimizer/inlinearray_bounds_check_tests.swift index 49e75f2340379..d586e3cc2ab06 100644 --- a/test/SILOptimizer/inlinearray_bounds_check_tests.swift +++ b/test/SILOptimizer/inlinearray_bounds_check_tests.swift @@ -262,7 +262,7 @@ public func inlinearray_binary_search_spl(_ v: InlineArray, // This prevents LoopRotate which prevent bounds checks opts since it depends on induction variable analysis which doesn't work on unrotated loops. // CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A35_sum_iterate_to_count_with_trap_splySis11InlineArrayVy$63_SiGF : // CHECK-SIL: bb2 -// CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" +// CHECK-NOT-SIL: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_br // CHECK-SIL-LABEL: } // end sil function '$s30inlinearray_bounds_check_tests0A35_sum_iterate_to_count_with_trap_splySis11InlineArrayVy$63_SiGF' public func inlinearray_sum_iterate_to_count_with_trap_spl(_ v: InlineArray<64, Int>) -> Int { @@ -326,7 +326,7 @@ public func inlinearray_inc_by_one(_ v: inout InlineArray, _ // CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A15_inc_by_one_splyys11InlineArrayVy$63_SiGz_SitF : // CHECK-SIL: bb2 -// CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" +// CHECK-NOT-SIL: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_br // CHECK-SIL-LABEL: } // end sil function '$s30inlinearray_bounds_check_tests0A15_inc_by_one_splyys11InlineArrayVy$63_SiGz_SitF' public func inlinearray_inc_by_one_spl(_ v: inout InlineArray<64, Int>, _ n: Int) { From d08d6707a68f117fe141c66a74239f92dec256f3 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Tue, 19 Aug 2025 15:05:54 +0100 Subject: [PATCH 38/44] Simplify cloner. --- .../InitializeStaticGlobals.swift | 2 +- .../LoopInvariantCodeMotion.swift | 11 +-- .../FunctionPasses/ObjectOutliner.swift | 6 +- .../SimplifyLoad.swift | 2 +- .../Sources/SIL/Utilities/Cloner.swift | 68 ++++++------------- 5 files changed, 33 insertions(+), 56 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift index aa2f33a712716..f6fdd00dbb493 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift @@ -225,7 +225,7 @@ private indirect enum GlobalInitValue { fatalError("cannot materialize undefined init value") case .constant(let value): - return cloner.cloneRecursively(value: value) + return cloner.cloneRecursivelyToGlobal(value: value) case .aggregate(let fields): if type.isStruct { diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 5b7abb51bc347..232f3d70f62ca 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -505,10 +505,13 @@ private extension MovableInstructions { var cloner = Cloner(cloneBefore: loop.preheader!.terminator, context) defer { cloner.deinitialize() } - guard let initialAddr = (cloner.cloneAddressProjections(addr: storeAddr) { srcAddr in - srcAddr.parentBlock.dominates(loop.preheader!, context.dominatorTree) - }) else { - return changed + let initialAddr = cloner.cloneRecursively(value: storeAddr) { srcAddr, cloner in + if srcAddr.parentBlock.dominates(loop.preheader!, context.dominatorTree) { + cloner.recordFoldedValue(srcAddr, mappedTo: srcAddr) + return srcAddr + } else { + return nil + } } let ownership: LoadInst.LoadOwnership = loop.preheader!.terminator.parentFunction.hasOwnership ? .trivial : .unqualified diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift index 9797b207b6e09..a704bca090955 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift @@ -369,7 +369,7 @@ private func constructObject(of allocRef: AllocRefInstBase, // Create the initializers for the fields var objectArgs = [Value]() for store in storesToClassFields { - objectArgs.append(cloner.cloneRecursively(value: store.source as! SingleValueInstruction)) + objectArgs.append(cloner.cloneRecursivelyToGlobal(value: store.source as! SingleValueInstruction)) } let globalBuilder = Builder(staticInitializerOf: global, context) @@ -381,7 +381,7 @@ private func constructObject(of allocRef: AllocRefInstBase, for elementIdx in 0.. { } return cloned } + + public mutating func cloneRecursivelyToGlobal(value: Value) -> Value { + return cloneRecursively(value: value) { value, cloner in + guard let beginAccess = value as? BeginAccessInst else { + return nil + } + + // Skip access instructions, which might be generated for UnsafePointer globals which point to other globals. + let clonedOperand = cloner.cloneRecursivelyToGlobal(value: beginAccess.address) + cloner.recordFoldedValue(beginAccess, mappedTo: clonedOperand) + return clonedOperand + } + } /// Transitively clones `value` including its defining instruction's operands. - public mutating func cloneRecursively(value: Value) -> Value { + public mutating func cloneRecursively(value: Value, checkBase: (Value, inout Cloner) -> Value?) -> Value { if isCloned(value: value) { return getClonedValue(of: value) } - if let beginAccess = value as? BeginAccessInst { - // Skip access instructions, which might be generated for UnsafePointer globals which point to other globals. - let clonedOperand = cloneRecursively(value: beginAccess.address) - bridged.recordFoldedValue(beginAccess.bridged, clonedOperand.bridged) - return clonedOperand + if let base = checkBase(value, &self) { + return base } guard let inst = value.definingInstruction else { @@ -84,7 +94,7 @@ public struct Cloner { } for op in inst.operands { - _ = cloneRecursively(value: op.value) + _ = cloneRecursively(value: op.value, checkBase: checkBase) } let cloned = clone(instruction: inst) @@ -96,46 +106,6 @@ public struct Cloner { fatalError("unexpected instruction kind") } - public mutating func cloneAddressProjections(addr: Value, checkBase: (Value) -> Bool) -> Value? { - // TODO: Temp fix - if addr is AllocStackInst { - return nil - } - - if checkBase(addr) { - bridged.recordFoldedValue(addr.bridged, addr.bridged) - return addr - } - - switch addr { - // The cloner does not currently know how to create compensating - // end_borrows or fix mark_dependence operands. - case is BeginBorrowInst, is MarkDependenceInst: return nil - case let singleValueInstruction as SingleValueInstruction: - guard let sourceOperand = singleValueInstruction.operands.first else { return nil } - - return cloneProjection(projectAddr: singleValueInstruction, sourceOperand: sourceOperand, checkBase: checkBase) - default: return nil - } - } - - private mutating func cloneProjection( - projectAddr: SingleValueInstruction, - sourceOperand: Operand, - checkBase: (Value) -> Bool - ) -> Value? { - guard let projectedSource = cloneAddressProjections( - addr: sourceOperand.value, - checkBase: checkBase - ) else { - return nil - } - - let clone = clone(instruction: projectAddr) - clone.setOperand(at: sourceOperand.index, to: projectedSource, context) - return clone as! SingleValueInstruction - } - public mutating func getClonedValue(of originalValue: Value) -> Value { bridged.getClonedValue(originalValue.bridged).value } @@ -147,4 +117,8 @@ public struct Cloner { public func getClonedBlock(for originalBlock: BasicBlock) -> BasicBlock { bridged.getClonedBasicBlock(originalBlock.bridged).block } + + public func recordFoldedValue(_ origValue: Value, mappedTo mappedValue: Value) { + bridged.recordFoldedValue(origValue.bridged, mappedValue.bridged) + } } From de9ec67daac81ea073270a9359b91927ff96c243 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Wed, 20 Aug 2025 15:41:38 +0100 Subject: [PATCH 39/44] Add stopCloning to cloner checkBase closure. Refactor. --- .../Sources/Optimizer/Analysis/LoopTree.swift | 3 + .../LoopInvariantCodeMotion.swift | 128 ++++++++++++++---- .../Sources/SIL/Utilities/Cloner.swift | 29 +++- 3 files changed, 127 insertions(+), 33 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index d961bafab565c..37e48366f45b7 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -39,6 +39,9 @@ struct Loop { } + exitingBlocks } + /// Exit blocks of the loop. + /// + /// - Note: Some exit blocks will be duplicated if the loop has critical edges. var exitBlocks: some Sequence { return loopBlocks.lazy .flatMap(\.successors) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 232f3d70f62ca..68fb3283f2a03 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -13,6 +13,7 @@ import SIL import OptimizerBridging +/// Hoist loop invariant code out of innermost loops. let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion") { function, context in for loop in context.loopTree.loops { optimizeTopLevelLoop(topLevelLoop: loop, context) @@ -23,9 +24,10 @@ private func optimizeTopLevelLoop(topLevelLoop: Loop, _ context: FunctionPassCon var innerLoops = Stack(context) defer { innerLoops.deinitialize() } - getWorkList(forLoop: topLevelLoop, result: &innerLoops) + getWorkList(forLoop: topLevelLoop, workList: &innerLoops) while let thisLoop = innerLoops.pop() { + // We only support Loops with a preheader. guard thisLoop.preheader != nil else { continue } @@ -39,14 +41,16 @@ private func optimizeTopLevelLoop(topLevelLoop: Loop, _ context: FunctionPassCon } } -private func getWorkList(forLoop loop: Loop, result: inout Stack) { - result.push(loop) +/// Creates post-order DFS work list of inner loops. +private func getWorkList(forLoop loop: Loop, workList: inout Stack) { + workList.push(loop) for innerLoop in loop.innerLoops { - getWorkList(forLoop: innerLoop, result: &result) + getWorkList(forLoop: innerLoop, workList: &workList) } } +/// Instructions that can be moved outside the loop. private struct MovableInstructions { var loadAndStoreAddrs: [AccessPath] = [] @@ -57,9 +61,12 @@ private struct MovableInstructions { var scopedInsts: [Instruction] = [] } +/// Analyzed instructions inside the currently processed loop. private struct AnalyzedInstructions { + /// Side effects of the loop. var loopSideEffects: StackWithCount private var blockSideEffectBottomMarker: StackWithCount.Marker + /// Side effects of the currently analyzed block. var sideEffectsOfCurrentBlock: StackWithCount.Segment { return StackWithCount.Segment( in: loopSideEffects, @@ -68,6 +75,9 @@ private struct AnalyzedInstructions { ) } + /// Contains either: + /// * an apply to the addressor of the global + /// * a builtin "once" of the global initializer var globalInitCalls: Stack var readOnlyApplies: Stack var loads: Stack @@ -75,8 +85,10 @@ private struct AnalyzedInstructions { var beginAccesses: Stack var fullApplies: Stack + /// `true` if the loop has instructions which (may) read from memory, which are not in `Loads` and not in `sideEffects`. var hasOtherMemReadingInsts = false + /// `true` if one of the side effects might release. lazy var sideEffectsMayRelease = loopSideEffects.contains(where: { $0.mayRelease }) init (_ context: FunctionPassContext) { @@ -101,6 +113,7 @@ private struct AnalyzedInstructions { fullApplies.deinitialize() } + /// Mark the start of currently processed block side effects. mutating func markBeginOfBlock() { blockSideEffectBottomMarker = loopSideEffects.top } @@ -131,6 +144,9 @@ private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext return movableInstructions } +/// Analyze instructions inside the `loop`. Compute side effects and populate `analyzedInstructions`. +/// +/// - note: Ideally, `movableInstructions` should be fully computed in `collectMovableInstructions`. private func analyzeInstructions( in loop: Loop, _ analyzedInstructions: inout AnalyzedInstructions, @@ -145,6 +161,7 @@ private func analyzeInstructions( if inst.hasOwnershipOperandsOrResults { analyzedInstructions.analyzeSideEffects(ofInst: inst) + // Collect fullApplies to be checked in analyzeBeginAccess if let fullApply = inst as? FullApplySite { analyzedInstructions.fullApplies.append(fullApply) } @@ -154,7 +171,7 @@ private func analyzeInstructions( switch inst { case is FixLifetimeInst: - break + break // We can ignore the side effects of FixLifetimes case let loadInst as LoadInst: analyzedInstructions.loads.append(loadInst) case let storeInst as StoreInst: @@ -177,14 +194,18 @@ private func analyzeInstructions( if applyInst.isSafeReadOnlyApply(context.calleeAnalysis) { analyzedInstructions.readOnlyApplies.append(applyInst) } else if let callee = applyInst.referencedFunction, - callee.isGlobalInitFunction, + callee.isGlobalInitFunction, // Calls to global inits are different because we don't care about side effects which are "after" the call in the loop. !applyInst.globalInitMayConflictWith( blockSideEffectSegment: analyzedInstructions.sideEffectsOfCurrentBlock, context.aliasAnalysis ) { + // Check against side-effects within the same block. + // Side-effects in other blocks are checked later (after we + // scanned all blocks of the loop). analyzedInstructions.globalInitCalls.append(applyInst) } + // Check for array semantics and side effects - same as default fallthrough default: switch inst { @@ -214,6 +235,7 @@ private func analyzeInstructions( } } +/// Process collected read only applies. Moves them to `hoistUp` if they don't conflict with any side effects. private func collectHoistableReadOnlyApplies( _ analyzedInstructions: AnalyzedInstructions, _ movableInstructions: inout MovableInstructions, @@ -233,6 +255,7 @@ private func collectHoistableReadOnlyApplies( } } +/// Process collected global init calls. Moves them to `hoistUp` if they don't conflict with any side effects. private func collectHoistableGlobalInitCalls( in loop: Loop, _ analyzedInstructions: AnalyzedInstructions, @@ -240,6 +263,7 @@ private func collectHoistableGlobalInitCalls( _ context: FunctionPassContext ) { for globalInitCall in analyzedInstructions.globalInitCalls { + // Check against side effects which are "before" (i.e. post-dominated by) the global initializer call. if !globalInitCall.globalInitMayConflictWith( loopSideEffects: analyzedInstructions.loopSideEffects, notPostDominatingPreheader: loop.preheader!, @@ -251,6 +275,8 @@ private func collectHoistableGlobalInitCalls( } } +/// Collect memory locations for which we can move all loads and stores out of the loop. +/// `loads` may mutate during this loop. private func collectProjectableAccessPathsAndSplitLoads( in loop: Loop, _ analyzedInstructions: inout AnalyzedInstructions, @@ -279,6 +305,7 @@ private func collectProjectableAccessPathsAndSplitLoads( } } +/// Computes movable instructions using computed analyzed instructions. private func collectMovableInstructions( in loop: Loop, _ analyzedInstructions: inout AnalyzedInstructions, @@ -324,6 +351,10 @@ private func collectMovableInstructions( } movableInstructions.loadsAndStores.append(storeInst) case let condFailInst as CondFailInst: + // We can (and must) hoist cond_fail instructions if the operand is + // invariant. We must hoist them so that we preserve memory safety. A + // cond_fail that would have protected (executed before) a memory access + // must - after hoisting - also be executed before said access. movableInstructions.hoistUp.append(condFailInst) case let beginAccessInst as BeginAccessInst: if beginAccessInst.canBeHoisted(outOf: loop, analyzedInstructions: analyzedInstructions, context) { @@ -336,6 +367,12 @@ private func collectMovableInstructions( } } +/// Optimizes the loop by performing in following order: +/// - speculative hoist +/// - projection, hoist and sink of loads and stores +/// - hoist of instructions that are guaranteed to be executed +/// - sink +/// - hoist with sink of scoped instructions private func optimizeLoop( loop: Loop, movableInstructions: inout MovableInstructions, @@ -354,6 +391,7 @@ private func optimizeLoop( } private extension AnalyzedInstructions { + /// Adds side effects of `inst` to the analyzed instructions. mutating func analyzeSideEffects(ofInst inst: Instruction) { if inst.mayHaveSideEffects { loopSideEffects.append(inst) @@ -362,6 +400,9 @@ private extension AnalyzedInstructions { } } + /// Find all loads that contain `accessPath`. Split them into a load with + /// identical `accessPath` and a set of non-overlapping loads. Add the new + /// non-overlapping loads to `loads`. mutating func splitLoads( storeAddr: Value, accessPath: AccessPath, @@ -375,6 +416,16 @@ private extension AnalyzedInstructions { var splitCounter = 0 while let loadInst = loads.pop() { + // Found a load wider than the store to accessPath. + // + // SplitLoads is called for each unique access path in the loop that is + // only loaded from and stored to and this loop takes time proportional to: + // num-wide-loads x num-fields x num-loop-memops + // + // For each load wider than the store, it creates a new load for each field + // in that type. Each new load is inserted in the LoadsAndStores vector. To + // avoid super-linear behavior for large types (e.g. giant tuples), limit + // growth of new loads to an arbitrary constant factor per access path. guard splitCounter <= 6 else { newLoads.push(loadInst) return false @@ -396,29 +447,36 @@ private extension AnalyzedInstructions { } private extension MovableInstructions { + /// Hoist instructions speculatively. + /// + /// Contrary to `hoistInstructions`, it doesn't only go through instructions in blocks that dominate all exits. mutating func speculativelyHoistInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { var changed = false + for inst in speculativelyHoistable { changed = inst.hoist(outOf: loop, context) || changed } + return changed } + /// Hoists and sinks loads with matching stores. Projects loads. mutating func hoistAndSinkLoadsAndStores(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { var changed = false + for accessPath in loadAndStoreAddrs { changed = hoistAndSinkLoadAndStore(outOf: loop, accessPath: accessPath, context: context) || changed } return changed } - + + /// Only hoists instructions in blocks that dominate all exit and latch blocks. + /// It doesn't hoist instructions speculatively. mutating func hoistInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(domTree: context.dominatorTree) var changed = false - // Only traverse instructions in blocks that dominate all exit and latch blocks. - // We don't hoist instructions speculatively here. for bb in dominatingBlocks { for inst in bb.instructions where hoistUp.contains(inst) { changed = inst.hoist(outOf: loop, context) || changed @@ -428,6 +486,7 @@ private extension MovableInstructions { return changed } + /// Sink instructions. mutating func sinkInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(domTree: context.dominatorTree) var changed = false @@ -439,6 +498,7 @@ private extension MovableInstructions { return changed } + /// Hoist and sink scoped instructions. mutating func hoistWithSinkScopedInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { guard !loop.hasNoExitBlocks else { return false @@ -471,10 +531,17 @@ private extension MovableInstructions { context: FunctionPassContext ) -> Bool { var changed = false + + // Initially load the value in the loop pre header. let builder = Builder(before: loop.preheader!.terminator, context) var storeAddr: Value? + // Set all stored values as available values in the ssaUpdater. + // If there are multiple stores in a block, only the last one counts. for case let storeInst as StoreInst in loadsAndStores where storeInst.storesTo(accessPath) { + // If a store just stores the loaded value, bail. The operand (= the load) + // will be removed later, so it cannot be used as available value. + // This corner case is surprisingly hard to handle, so we just give up. if let srcLoadInst = storeInst.source as? LoadInst, srcLoadInst.loadsFrom(accessPath) { return changed @@ -483,6 +550,10 @@ private extension MovableInstructions { if storeAddr == nil { storeAddr = storeInst.destination } else if storeInst.destination.type != storeAddr!.type { + // This transformation assumes that the values of all stores in the loop + // must be interchangeable. It won't work if stores different types + // because of casting or payload extraction even though they have the + // same access path. return changed } } @@ -505,13 +576,23 @@ private extension MovableInstructions { var cloner = Cloner(cloneBefore: loop.preheader!.terminator, context) defer { cloner.deinitialize() } - let initialAddr = cloner.cloneRecursively(value: storeAddr) { srcAddr, cloner in + guard let initialAddr = (cloner.cloneRecursively(value: storeAddr) { srcAddr, cloner in + switch srcAddr { + case is AllocStackInst, is BeginBorrowInst, is MarkDependenceInst: + return .stopCloning + default: break + } + + // Clone projections until the address dominates preheader. if srcAddr.parentBlock.dominates(loop.preheader!, context.dominatorTree) { cloner.recordFoldedValue(srcAddr, mappedTo: srcAddr) - return srcAddr + return .setBase(srcAddr) } else { - return nil + // Return nil invalid to continue cloning. + return .continueCloning } + }) else { + return changed } let ownership: LoadInst.LoadOwnership = loop.preheader!.terminator.parentFunction.hasOwnership ? .trivial : .unqualified @@ -522,8 +603,9 @@ private extension MovableInstructions { var currentBlock: BasicBlock? var currentVal: Value? - // This loop depends on loadsAndStores being - // in order the instructions appear in blocks! + // Remove all stores and replace the loads with the current value. + // + // This loop depends on loadsAndStores being in order the instructions appear in blocks. for inst in loadsAndStores { let block = inst.parentBlock @@ -544,6 +626,7 @@ private extension MovableInstructions { continue } + // If we didn't see a store in this block yet, get the current value from the ssaUpdater. let rootVal = currentVal ?? ssaUpdater.getValue(inMiddleOf: block) if loadInst.operand.value.accessPath == accessPath { @@ -565,6 +648,7 @@ private extension MovableInstructions { loadsAndStores.removeAll(where: { $0.isDeleted }) + // Store back the value at all loop exits. for exitingOrLatchBlock in loop.exitingAndLatchBlocks { for succesor in exitingOrLatchBlock.successors where !loop.contains(block: succesor) { guard let firstInst = succesor.instructions.first else { @@ -583,6 +667,7 @@ private extension MovableInstructions { } } + // In case the value is only stored but never loaded in the loop. if initialLoad.uses.isEmpty { context.erase(instruction: initialLoad) } @@ -702,15 +787,10 @@ private extension Instruction { var canMove = loop.hasSingleExitBlock let exitBlocks = loop.exitBlocks let exitingBlocks = loop.exitingBlocks - var newExitBlocks = Stack(context) - defer { newExitBlocks.deinitialize() } var changed = false for exitingBlock in exitingBlocks { - for succesor in exitingBlock.successors where !newExitBlocks.contains(succesor) && exitBlocks.contains(succesor) { - - newExitBlocks.push(succesor) - + for succesor in exitingBlock.successors where exitBlocks.contains(succesor) { if succesor.instructions.contains(where: { isIdenticalTo($0) }) { canMove = false } else if canMove { @@ -794,13 +874,6 @@ private extension Instruction { ) } } - - func isInAccessScope(of beginAccess: BeginAccessInst, _ domTree: DominatorTree) -> Bool { - return beginAccess.dominates(self, domTree) && beginAccess.endAccessInstructions - .allSatisfy { endAccessInst in - self.dominates(endAccessInst, domTree) - } - } } private extension UnaryInstruction { @@ -900,6 +973,7 @@ private extension ApplyInst { } private extension BeginAccessInst { + /// Returns `true` if this begin access is safe to hoist. func canBeHoisted( outOf loop: Loop, analyzedInstructions: AnalyzedInstructions, diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift index d491ca06864ec..b77ff3e8cbed1 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift @@ -22,6 +22,12 @@ public struct Cloner { public var bridged: BridgedCloner public let context: Context + public enum CheckBaseResult { + case continueCloning + case setBase(Value) + case stopCloning + } + public enum Target { case function(Function) case global(GlobalVariable) @@ -67,26 +73,35 @@ public struct Cloner { } public mutating func cloneRecursivelyToGlobal(value: Value) -> Value { - return cloneRecursively(value: value) { value, cloner in + guard let cloned = (cloneRecursively(value: value) { value, cloner in guard let beginAccess = value as? BeginAccessInst else { - return nil + return .continueCloning } // Skip access instructions, which might be generated for UnsafePointer globals which point to other globals. let clonedOperand = cloner.cloneRecursivelyToGlobal(value: beginAccess.address) cloner.recordFoldedValue(beginAccess, mappedTo: clonedOperand) - return clonedOperand + return .setBase(clonedOperand) + }) else { + fatalError("Clone recursively to global shouldn't bail.") } + + return cloned } /// Transitively clones `value` including its defining instruction's operands. - public mutating func cloneRecursively(value: Value, checkBase: (Value, inout Cloner) -> Value?) -> Value { + public mutating func cloneRecursively(value: Value, checkBase: (Value, inout Cloner) -> CheckBaseResult) -> Value? { if isCloned(value: value) { return getClonedValue(of: value) } - if let base = checkBase(value, &self) { + switch checkBase(value, &self) { + case .setBase(let base): return base + case .stopCloning: + return nil + case .continueCloning: + break } guard let inst = value.definingInstruction else { @@ -94,7 +109,9 @@ public struct Cloner { } for op in inst.operands { - _ = cloneRecursively(value: op.value, checkBase: checkBase) + if cloneRecursively(value: op.value, checkBase: checkBase) == nil { + return nil + } } let cloned = clone(instruction: inst) From f1b150216a64e710890cba18de1322bb0a0e8846 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Thu, 21 Aug 2025 17:31:49 +0100 Subject: [PATCH 40/44] Refactor. --- .../LoopInvariantCodeMotion.swift | 312 +++++++++--------- 1 file changed, 147 insertions(+), 165 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 68fb3283f2a03..309d1e378df47 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -52,7 +52,7 @@ private func getWorkList(forLoop loop: Loop, workList: inout Stack) { /// Instructions that can be moved outside the loop. private struct MovableInstructions { - var loadAndStoreAddrs: [AccessPath] = [] + var loadAndStoreAccessPaths: [AccessPath] = [] var speculativelyHoistable: [Instruction] = [] var loadsAndStores: [Instruction] = [] @@ -264,12 +264,12 @@ private func collectHoistableGlobalInitCalls( ) { for globalInitCall in analyzedInstructions.globalInitCalls { // Check against side effects which are "before" (i.e. post-dominated by) the global initializer call. - if !globalInitCall.globalInitMayConflictWith( - loopSideEffects: analyzedInstructions.loopSideEffects, - notPostDominatingPreheader: loop.preheader!, - context.aliasAnalysis, - context.postDominatorTree - ) { + if globalInitCall.parentBlock.postDominates(loop.preheader!, context.postDominatorTree), + !globalInitCall.globalInitMayConflictWith( + loopSideEffects: analyzedInstructions.loopSideEffects, + context.aliasAnalysis, + context.postDominatorTree + ) { movableInstructions.hoistUp.append(globalInitCall) } } @@ -287,19 +287,19 @@ private func collectProjectableAccessPathsAndSplitLoads( for storeInst in analyzedInstructions.stores { let accessPath = storeInst.destination.accessPath if accessPath.isLoopInvariant(loop: loop), - accessPath.isOnlyLoadedAndStored( - storeAddr: storeInst.destination, - analyzedInstructions: analyzedInstructions, - context.aliasAnalysis + analyzedInstructions.isOnlyLoadedAndStored( + accessPath: accessPath, + storeAddr: storeInst.destination, + context.aliasAnalysis ), - !movableInstructions.loadAndStoreAddrs.contains(accessPath), - loop.storesCommonlyDominateExits(analyzedInstructions: analyzedInstructions, accessPath: accessPath, context), + !movableInstructions.loadAndStoreAccessPaths.contains(accessPath), + analyzedInstructions.storesCommonlyDominateExits(of: loop, storingTo: accessPath, context), analyzedInstructions.splitLoads( - storeAddr: storeInst.destination, - accessPath: accessPath, - context + storeAddr: storeInst.destination, + accessPath: accessPath, + context ) { - movableInstructions.loadAndStoreAddrs.append(accessPath) + movableInstructions.loadAndStoreAccessPaths.append(accessPath) } } } @@ -326,16 +326,14 @@ private func collectMovableInstructions( continue } - if !analyzedInstructions.sideEffectsMayRelease || !fixLifetimeInst.mayWriteTo( - sideEffects: analyzedInstructions.loopSideEffects, - context.aliasAnalysis - ) { + if !analyzedInstructions.sideEffectsMayRelease || !analyzedInstructions.loopSideEffectsMayWriteTo(address: fixLifetimeInst.operand.value, context.aliasAnalysis) { movableInstructions.sinkDown.append(fixLifetimeInst) } case let loadInst as LoadInst: // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. if loadInstCounter * analyzedInstructions.loopSideEffects.count < 8000, - !loadInst.mayWriteTo(sideEffects: analyzedInstructions.loopSideEffects, context.aliasAnalysis) { + !analyzedInstructions.loopSideEffectsMayWriteTo(address: loadInst.operand.value, context.aliasAnalysis), + loadInst.loadOwnership != .take, loadInst.loadOwnership != .copy { // TODO: Add support for take and copy loads. movableInstructions.hoistUp.append(loadInst) } @@ -390,6 +388,21 @@ private func optimizeLoop( return changed } +extension BasicBlock { + func containsStoresTo(accessPath: AccessPath) -> Bool { + return instructions.contains { inst in + return inst.operands.contains { operand in + if let storeInst = operand.instruction as? StoreInst, + storeInst.destination.accessPath == accessPath { + return true + } else { + return false + } + } + } + } +} + private extension AnalyzedInstructions { /// Adds side effects of `inst` to the analyzed instructions. mutating func analyzeSideEffects(ofInst inst: Instruction) { @@ -400,6 +413,99 @@ private extension AnalyzedInstructions { } } + /// Returns true if all instructions in `sideEffects` which may alias with + /// this path are either loads or stores from this path. + /// + /// `storeAddr` is only needed for AliasAnalysis until we have an interface + /// that supports `AccessPath`. + func isOnlyLoadedAndStored( + accessPath: AccessPath, + storeAddr: Value, + _ aliasAnalysis: AliasAnalysis + ) -> Bool { + if (loopSideEffects.contains { sideEffect in + switch sideEffect { + case let storeInst as StoreInst: + if storeInst.storesTo(accessPath) { + return false + } + case let loadInst as LoadInst: + if loadInst.loadsFrom(accessPath) { + return false + } + default: break + } + + return sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) + }) { + return false + } + + if (loads.contains { loadInst in + loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) && loadInst.loadOwnership != .take && !loadInst.overlaps(accessPath: accessPath) + }) { + return false + } + + if (stores.contains { storeInst in + storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && !storeInst.storesTo(accessPath) + }) { + return false + } + + return true + } + + /// Returns `true` if all stores to `accessPath` commonly dominate the loop exits. + func storesCommonlyDominateExits(of loop: Loop, storingTo accessPath: AccessPath, _ context: FunctionPassContext) -> Bool { + var exitingBlocksSet = BasicBlockSet(context) + var storeBlocks = BasicBlockSet(context) + var worklist = BasicBlockWorklist(context) + defer { + exitingBlocksSet.deinitialize() + storeBlocks.deinitialize() + worklist.deinitialize() + } + + if loop.preheader!.containsStoresTo(accessPath: accessPath) { + return true + } + + exitingBlocksSet.insert(contentsOf: loop.exitingBlocks) + + storeBlocks.insert(contentsOf: stores + .filter({ $0.destination.accessPath == accessPath }) + .map(\.parentBlock)) + + if storeBlocks.contains(loop.header) { + return true + } + + worklist.pushIfNotVisited(loop.header) + while let block = worklist.pop() { + if storeBlocks.contains(block) { + continue + } + + if exitingBlocksSet.contains(block) && block.successors.count(where: { $0.terminator is UnreachableInst }) != block.successors.count { + return false + } + + worklist.pushIfNotVisited(contentsOf: block.successors) + } + + return true + } + + /// Returns true if `loopSideEffects` contains any memory writes which + /// may alias with the memory `address`. + func loopSideEffectsMayWriteTo(address: Value, _ aliasAnalysis: AliasAnalysis) -> Bool { + return loopSideEffects + .contains { sideEffect in + sideEffect.mayWrite(toAddress: address, aliasAnalysis) + } + } + /// Find all loads that contain `accessPath`. Split them into a load with /// identical `accessPath` and a set of non-overlapping loads. Add the new /// non-overlapping loads to `loads`. @@ -464,7 +570,7 @@ private extension MovableInstructions { mutating func hoistAndSinkLoadsAndStores(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { var changed = false - for accessPath in loadAndStoreAddrs { + for accessPath in loadAndStoreAccessPaths { changed = hoistAndSinkLoadAndStore(outOf: loop, accessPath: accessPath, context: context) || changed } @@ -565,7 +671,7 @@ private extension MovableInstructions { var ssaUpdater = SSAUpdater( function: storeAddr.parentFunction, type: storeAddr.type.objectType, - ownership: storeAddr.ownership, + ownership: .none, context ) @@ -676,51 +782,6 @@ private extension MovableInstructions { } } -private extension Loop { - /// Returns `true` if all stores to `accessPath` commonly dominate the loop exits. - func storesCommonlyDominateExits( - analyzedInstructions: AnalyzedInstructions, - accessPath: AccessPath, - _ context: FunctionPassContext - ) -> Bool { - var storeBlocks = BasicBlockSet(context) - var worklist = BasicBlockWorklist(context) - defer { - storeBlocks.deinitialize() - worklist.deinitialize() - } - - if preheader!.instructions.contains(where: { $0.operands.contains(where: { operand in - operand.instruction is StoreInst && operand.value.accessPath == accessPath - })}) { - return true - } - - storeBlocks.insert(contentsOf: analyzedInstructions.stores - .filter({ $0.destination.accessPath == accessPath }) - .map(\.parentBlock)) - - if storeBlocks.contains(header) { - return true - } - - worklist.pushIfNotVisited(header) - while let block = worklist.pop() { - if storeBlocks.contains(block) { - continue - } - - if exitingBlocks.contains(block) && !block.successors.contains(where: {$0.terminator is UnreachableInst}) { - return false - } - - worklist.pushIfNotVisited(contentsOf: block.successors) - } - - return true - } -} - private extension Instruction { var hasOwnershipOperandsOrResults: Bool { guard parentFunction.hasOwnership else { return false } @@ -729,7 +790,8 @@ private extension Instruction { || operands.contains(where: { $0.value.ownership != .none }) } - /// Returns `true` if this instruction follows the default hoisting heuristic. + /// Returns `true` if this instruction follows the default hoisting heuristic which means it + /// is not a terminator, allocation or deallocation and either a hoistable array semantics call or doesn't have memory effects. func canBeHoisted(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { switch self { case is TermInst, is Allocation, is Deallocation: @@ -784,28 +846,23 @@ private extension Instruction { } func sink(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { - var canMove = loop.hasSingleExitBlock let exitBlocks = loop.exitBlocks - let exitingBlocks = loop.exitingBlocks var changed = false - for exitingBlock in exitingBlocks { - for succesor in exitingBlock.successors where exitBlocks.contains(succesor) { - if succesor.instructions.contains(where: { isIdenticalTo($0) }) { - canMove = false - } else if canMove { - move(before: succesor.instructions.first!, context) - } else if let firstInstruction = succesor.instructions.first { - copy(before: firstInstruction, context) - } else { - continue - } - + for exitBlock in exitBlocks { + if exitBlock.instructions.contains(where: { isIdenticalTo($0) }) { + continue + } + + if changed { + copy(before: exitBlock.instructions.first!, context) + } else { + move(before: exitBlock.instructions.first!, context) changed = true } } - - if changed && !canMove { + + if !changed { context.erase(instruction: self) } @@ -851,41 +908,17 @@ private extension Instruction { /// the call. func globalInitMayConflictWith( loopSideEffects: StackWithCount, - notPostDominatingPreheader: BasicBlock, _ aliasAnalysis: AliasAnalysis, _ postDomTree: PostDominatorTree ) -> Bool { - guard parentBlock.postDominates(notPostDominatingPreheader, postDomTree) else { - return true - } - return loopSideEffects .contains { sideEffect in // Only check instructions in blocks which are "before" (i.e. post-dominated // by) the block which contains the init-call. // Instructions which are before the call in the same block have already // been checked. - parentBlock.strictlyPostDominates( - sideEffect.parentBlock, - postDomTree - ) && globalInitMayConflictWith( - sideEffect: sideEffect, - aliasAnalysis - ) - } - } -} - -private extension UnaryInstruction { - /// Returns true if `sideEffects` contains any memory writes which - /// may alias with the memory addressed by this instruction. - func mayWriteTo( - sideEffects: StackWithCount, - _ aliasAnalysis: AliasAnalysis - ) -> Bool { - return sideEffects - .contains { sideEffect in - sideEffect.mayWrite(toAddress: operand.value, aliasAnalysis) + parentBlock.strictlyPostDominates(sideEffect.parentBlock, postDomTree) && + globalInitMayConflictWith(sideEffect: sideEffect, aliasAnalysis) } } } @@ -909,10 +942,6 @@ private extension LoadInst { } func overlaps(accessPath: AccessPath) -> Bool { - guard self.loadOwnership != .take else { // TODO: handle LoadOwnershipQualifier::Take - return false - } - return accessPath.isEqualOrContains(self.operand.value.accessPath) || self.operand.value.accessPath.isEqualOrContains(accessPath) } } @@ -996,14 +1025,8 @@ private extension BeginAccessInst { defer { scope.deinitialize() } for fullApplyInst in analyzedInstructions.fullApplies { - guard mayWriteToMemory - ? fullApplyInst.mayReadOrWrite( - address: address, - context.aliasAnalysis - ) : fullApplyInst.mayWrite( - toAddress: address, - context.aliasAnalysis - ) else { + guard mayWriteToMemory && fullApplyInst.mayReadOrWrite(address: address, context.aliasAnalysis) || + !mayWriteToMemory && fullApplyInst.mayWrite(toAddress: address, context.aliasAnalysis) else { continue } @@ -1050,45 +1073,4 @@ private extension AccessPath { return projectionPath.isConstant } - - /// Returns true if all instructions in `sideEffects` which may alias with - /// this path are either loads or stores from this path. - /// - /// `storeAddr` is only needed for AliasAnalysis until we have an interface - /// that supports `AccessPath`. - func isOnlyLoadedAndStored( - storeAddr: Value, - analyzedInstructions: AnalyzedInstructions, - _ aliasAnalysis: AliasAnalysis - ) -> Bool { - if (analyzedInstructions.loopSideEffects.contains { sideEffect in - let accessesPath: Bool - switch sideEffect { - case let storeInst as StoreInst: - accessesPath = storeInst.storesTo(self) - case let loadInst as LoadInst: - accessesPath = loadInst.loadsFrom(self) - default: - accessesPath = false - } - - return !accessesPath && sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) - }) { - return false - } - - if (analyzedInstructions.loads.contains { loadInst in - loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) && !loadInst.overlaps(accessPath: self) - }) { - return false - } - - if (analyzedInstructions.stores.contains { storeInst in - storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && !storeInst.storesTo(self) - }) { - return false - } - - return true - } } From 15e9365b056cd9f329a8e5475660e8f0039f3e17 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Wed, 27 Aug 2025 13:17:42 +0100 Subject: [PATCH 41/44] Move remaining comments from the old to new implementation. Refactor. --- .../Sources/Optimizer/Analysis/LoopTree.swift | 2 +- .../LoopInvariantCodeMotion.swift | 113 ++++++++++++------ .../Sources/SIL/Utilities/Cloner.swift | 22 ++-- 3 files changed, 91 insertions(+), 46 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index 37e48366f45b7..b49f067ad4fa3 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -107,7 +107,7 @@ struct Loop { } } - func splitCriticalEdges(_ context: FunctionPassContext) { + func splitCriticalExitingAndBackEdges(_ context: FunctionPassContext) { for exitingOrLatchBlock in exitingAndLatchBlocks { for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !contains(block: succesor) { splitCriticalEdge( diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 309d1e378df47..66330571bc0c0 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -125,7 +125,7 @@ private struct AnalyzedInstructions { /// This may split some loads into smaller loads. private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext) -> MovableInstructions { // TODO: Remove once uses lowered OSSA. - loop.splitCriticalEdges(context) + loop.splitCriticalExitingAndBackEdges(context) var movableInstructions = MovableInstructions() var analyzedInstructions = AnalyzedInstructions(context) @@ -293,6 +293,8 @@ private func collectProjectableAccessPathsAndSplitLoads( context.aliasAnalysis ), !movableInstructions.loadAndStoreAccessPaths.contains(accessPath), + // This is not a requirement for functional correctness, but we don't want to + // _speculatively_ load and store the value (outside of the loop). analyzedInstructions.storesCommonlyDominateExits(of: loop, storingTo: accessPath, context), analyzedInstructions.splitLoads( storeAddr: storeInst.destination, @@ -426,7 +428,7 @@ private extension AnalyzedInstructions { if (loopSideEffects.contains { sideEffect in switch sideEffect { case let storeInst as StoreInst: - if storeInst.storesTo(accessPath) { + if storeInst.isNonInitializingStoreTo(accessPath) { return false } case let loadInst as LoadInst: @@ -436,6 +438,7 @@ private extension AnalyzedInstructions { default: break } + // Pass the original address value until we can fix alias analysis. return sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) }) { return false @@ -448,7 +451,7 @@ private extension AnalyzedInstructions { } if (stores.contains { storeInst in - storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && !storeInst.storesTo(accessPath) + storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && !storeInst.isNonInitializingStoreTo(accessPath) }) { return false } @@ -467,20 +470,56 @@ private extension AnalyzedInstructions { worklist.deinitialize() } + // Also a store in the pre-header dominates all exists. Although the situation + // is a bit different here: the store in the pre-header remains - it's not + // (re)moved by the LICM transformation. + // But even if the loop-stores are not dominating the loop exits, it + // makes sense to move them out of the loop if this case. When this is done, + // dead-store-elimination can then most likely eliminate the store in the + // pre-header. + // + // pre_header: + // store %v1 to %addr + // header: + // cond_br %cond, then, tail + // then: + // store %v2 to %addr // a conditional store in the loop + // br tail + // tail: + // cond_br %loop_cond, header, exit + // exit: + // + // will be transformed to + // + // pre_header: + // store %v1 to %addr // <- can be removed by DSE afterwards + // header: + // cond_br %cond, then, tail + // then: + // br tail + // tail(%phi): + // cond_br %loop_cond, header, exit + // exit: + // store %phi to %addr + // if loop.preheader!.containsStoresTo(accessPath: accessPath) { return true } + // Create a set of exiting blocks for efficient lookup later. exitingBlocksSet.insert(contentsOf: loop.exitingBlocks) + // Collect as many recognizable store parent blocks as possible. It's ok if not all stores are collected. storeBlocks.insert(contentsOf: stores .filter({ $0.destination.accessPath == accessPath }) .map(\.parentBlock)) + // If a store is in the loop header, we already know that it's dominating all loop exits. if storeBlocks.contains(loop.header) { return true } + // Starting from the header, check whether all stores are alive. worklist.pushIfNotVisited(loop.header) while let block = worklist.pop() { if storeBlocks.contains(block) { @@ -621,8 +660,14 @@ private extension MovableInstructions { continue } + // We only want to sink the first end_access and erase the rest to not introduce duplicates. + var sankFirst = false for endAccess in beginAccessInst.endAccessInstructions { - _ = endAccess.sink(outOf: loop, context) + if sankFirst { + context.erase(instruction: endAccess) + } else { + sankFirst = endAccess.sink(outOf: loop, context) + } } changed = true @@ -644,7 +689,7 @@ private extension MovableInstructions { // Set all stored values as available values in the ssaUpdater. // If there are multiple stores in a block, only the last one counts. - for case let storeInst as StoreInst in loadsAndStores where storeInst.storesTo(accessPath) { + for case let storeInst as StoreInst in loadsAndStores where storeInst.isNonInitializingStoreTo(accessPath) { // If a store just stores the loaded value, bail. The operand (= the load) // will be removed later, so it cannot be used as available value. // This corner case is surprisingly hard to handle, so we just give up. @@ -675,7 +720,7 @@ private extension MovableInstructions { context ) - for case let storeInst as StoreInst in loadsAndStores where storeInst.storesTo(accessPath) { + for case let storeInst as StoreInst in loadsAndStores where storeInst.isNonInitializingStoreTo(accessPath) { ssaUpdater.addAvailableValue(storeInst.source, in: storeInst.parentBlock) } @@ -692,10 +737,10 @@ private extension MovableInstructions { // Clone projections until the address dominates preheader. if srcAddr.parentBlock.dominates(loop.preheader!, context.dominatorTree) { cloner.recordFoldedValue(srcAddr, mappedTo: srcAddr) - return .setBase(srcAddr) + return .customValue(srcAddr) } else { // Return nil invalid to continue cloning. - return .continueCloning + return .defaultValue } }) else { return changed @@ -720,7 +765,7 @@ private extension MovableInstructions { currentVal = nil } - if let storeInst = inst as? StoreInst, storeInst.storesTo(accessPath) { + if let storeInst = inst as? StoreInst, storeInst.isNonInitializingStoreTo(accessPath) { currentVal = storeInst.source context.erase(instruction: storeInst) changed = true @@ -755,22 +800,18 @@ private extension MovableInstructions { loadsAndStores.removeAll(where: { $0.isDeleted }) // Store back the value at all loop exits. - for exitingOrLatchBlock in loop.exitingAndLatchBlocks { - for succesor in exitingOrLatchBlock.successors where !loop.contains(block: succesor) { - guard let firstInst = succesor.instructions.first else { - continue - } - - let builder = Builder(before: firstInst, context) - - let ownership: StoreInst.StoreOwnership = firstInst.parentFunction.hasOwnership ? .trivial : .unqualified - builder.createStore( - source: ssaUpdater.getValue(inMiddleOf: succesor), - destination: initialAddr, - ownership: ownership - ) - changed = true - } + for exitBlock in loop.exitBlocks { + assert(exitBlock.hasSinglePredecessor, "Exiting edge should not be critical.") + + let builder = Builder(before: exitBlock.instructions.first!, context) + + let ownership: StoreInst.StoreOwnership = exitBlock.instructions.first!.parentFunction.hasOwnership ? .trivial : .unqualified + builder.createStore( + source: ssaUpdater.getValue(inMiddleOf: exitBlock), + destination: initialAddr, + ownership: ownership + ) + changed = true } // In case the value is only stored but never loaded in the loop. @@ -850,9 +891,7 @@ private extension Instruction { var changed = false for exitBlock in exitBlocks { - if exitBlock.instructions.contains(where: { isIdenticalTo($0) }) { - continue - } + assert(exitBlock.hasSinglePredecessor, "Exiting edge should not be critical.") if changed { copy(before: exitBlock.instructions.first!, context) @@ -861,10 +900,6 @@ private extension Instruction { changed = true } } - - if !changed { - context.erase(instruction: self) - } return changed } @@ -925,8 +960,8 @@ private extension Instruction { private extension StoreInst { /// Returns a `true` if this store is a store to `accessPath`. - func storesTo(_ accessPath: AccessPath) -> Bool { - guard self.storeOwnership != .initialize else { + func isNonInitializingStoreTo(_ accessPath: AccessPath) -> Bool { + if self.storeOwnership == .initialize { return false } @@ -942,11 +977,13 @@ private extension LoadInst { } func overlaps(accessPath: AccessPath) -> Bool { + // Don't use `AccessPath.mayOverlap`. We only want definite overlap. return accessPath.isEqualOrContains(self.operand.value.accessPath) || self.operand.value.accessPath.isEqualOrContains(accessPath) } } private extension ApplyInst { + /// Returns `true` if this apply inst could be safely hoisted. func isSafeReadOnlyApply(_ calleeAnalysis: CalleeAnalysis) -> Bool { guard functionConvention.results.allSatisfy({ $0.convention == .unowned }) else { return false @@ -960,6 +997,9 @@ private extension ApplyInst { return !calleeAnalysis.getSideEffects(ofApply: self).memory.write } + /// Returns `true` if the `sideEffects` contain any memory writes which + /// may alias with any memory which is read by this `ApplyInst`. + /// - Note: This function should only be called on a read-only apply! func isSafeReadOnlyApply( for sideEffects: StackWithCount, _ aliasAnalysis: AliasAnalysis, @@ -969,6 +1009,7 @@ private extension ApplyInst { return true } + // Check if the memory addressed by the argument may alias any writes. for sideEffect in sideEffects { switch sideEffect { case let storeInst as StoreInst: @@ -1030,6 +1071,7 @@ private extension BeginAccessInst { continue } + // After hoisting the begin/end_access the apply will be within the scope, so it must not have a conflicting access. if !scope.contains(fullApplyInst) { return false } @@ -1038,6 +1080,8 @@ private extension BeginAccessInst { switch accessPath.base { case .class, .global: for sideEffect in analyzedInstructions.loopSideEffects where sideEffect.mayRelease { + // Since a class might have a deinitializer, hoisting begin/end_access pair could violate + // exclusive access if the deinitializer accesses address used by begin_access. if !scope.contains(sideEffect) { return false } @@ -1051,6 +1095,7 @@ private extension BeginAccessInst { } private extension AccessPath { + /// Returns `true` if this access path is invariant in `loop`. func isLoopInvariant(loop: Loop) -> Bool { switch base { case .box(let inst as Instruction), .class(let inst as Instruction), diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift index b77ff3e8cbed1..ffdf7b1e94dec 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift @@ -22,9 +22,9 @@ public struct Cloner { public var bridged: BridgedCloner public let context: Context - public enum CheckBaseResult { - case continueCloning - case setBase(Value) + public enum GetClonedResult { + case defaultValue + case customValue(Value) case stopCloning } @@ -73,15 +73,15 @@ public struct Cloner { } public mutating func cloneRecursivelyToGlobal(value: Value) -> Value { - guard let cloned = (cloneRecursively(value: value) { value, cloner in + guard let cloned = cloneRecursively(value: value, customGetCloned: { value, cloner in guard let beginAccess = value as? BeginAccessInst else { - return .continueCloning + return .defaultValue } // Skip access instructions, which might be generated for UnsafePointer globals which point to other globals. let clonedOperand = cloner.cloneRecursivelyToGlobal(value: beginAccess.address) cloner.recordFoldedValue(beginAccess, mappedTo: clonedOperand) - return .setBase(clonedOperand) + return .customValue(clonedOperand) }) else { fatalError("Clone recursively to global shouldn't bail.") } @@ -90,17 +90,17 @@ public struct Cloner { } /// Transitively clones `value` including its defining instruction's operands. - public mutating func cloneRecursively(value: Value, checkBase: (Value, inout Cloner) -> CheckBaseResult) -> Value? { + public mutating func cloneRecursively(value: Value, customGetCloned: (Value, inout Cloner) -> GetClonedResult) -> Value? { if isCloned(value: value) { return getClonedValue(of: value) } - switch checkBase(value, &self) { - case .setBase(let base): + switch customGetCloned(value, &self) { + case .customValue(let base): return base case .stopCloning: return nil - case .continueCloning: + case .defaultValue: break } @@ -109,7 +109,7 @@ public struct Cloner { } for op in inst.operands { - if cloneRecursively(value: op.value, checkBase: checkBase) == nil { + if cloneRecursively(value: op.value, customGetCloned: customGetCloned) == nil { return nil } } From 7e27996af06c79246dd63981b87896437fa122ab Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Wed, 27 Aug 2025 13:38:48 +0100 Subject: [PATCH 42/44] Fix merge artifacts. --- .../ClosureSpecialization.swift | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift index 1c66b8499b5cc..09f656f10d15e 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift @@ -247,8 +247,8 @@ private func getOrCreateSpecializedFunction( context.buildSpecializedFunction( specializedFunction: specializedFunction, buildFn: { (emptySpecializedFunction, functionPassContext) in - let closureSpecCloner = SpecializationCloner( - emptySpecializedFunction: emptySpecializedFunction, functionPassContext) + let closureSpecCloner = Cloner( + cloneToEmptyFunction: emptySpecializedFunction, functionPassContext) closureSpecCloner.cloneAndSpecializeFunctionBody(using: pullbackClosureInfo) }) @@ -731,15 +731,14 @@ private func markConvertedAndReabstractedClosuresAsUsed( } } -extension SpecializationCloner { +extension Cloner where Context == FunctionPassContext { fileprivate func cloneAndSpecializeFunctionBody(using pullbackClosureInfo: PullbackClosureInfo) { self.cloneEntryBlockArgsWithoutOrigClosures(usingOrigCalleeAt: pullbackClosureInfo) let (allSpecializedEntryBlockArgs, closureArgIndexToAllClonedReleasableClosures) = cloneAllClosures(at: pullbackClosureInfo) - self.cloneFunctionBody( - from: pullbackClosureInfo.pullbackFn, entryBlockArguments: allSpecializedEntryBlockArgs) + self.cloneFunctionBody(from: pullbackClosureInfo.pullbackFn, entryBlockArguments: allSpecializedEntryBlockArgs) self.insertCleanupCodeForClonedReleasableClosures( from: pullbackClosureInfo, @@ -824,8 +823,8 @@ extension SpecializationCloner { { let (origToClonedValueMap, capturedArgRange) = self.addEntryBlockArgs( forValuesCapturedBy: closureArgDesc) - let clonedFunction = self.cloned - let clonedEntryBlock = self.entryBlock + let clonedFunction = self.targetFunction + let clonedEntryBlock = self.getOrCreateEntryBlock() let clonedClosureArgs = Array(clonedEntryBlock.arguments[capturedArgRange]) let builder = @@ -909,8 +908,8 @@ extension SpecializationCloner { } } - if self.context.needFixStackNesting { - self.context.fixStackNesting(in: self.cloned) + if (self.context.needFixStackNesting) { + self.context.fixStackNesting(in: targetFunction) } } } @@ -1426,12 +1425,10 @@ private struct PullbackClosureInfo { } func specializedCalleeName(_ context: FunctionPassContext) -> String { - let closureArgs = Array(self.closureArgDescriptors.map { $0.closure }) - let closureIndices = Array(self.closureArgDescriptors.map { $0.closureArgIndex }) - - return context.mangle( - withClosureArguments: closureArgs, closureArgIndices: closureIndices, - from: pullbackFn) + let closureArgs = Array(self.closureArgDescriptors.map { + (argumentIndex: $0.closureArgIndex, argumentValue: $0.closure) + }) + return context.mangle(withClosureArguments: closureArgs, from: pullbackFn) } } From 71f9b050ce812c02382a3b6121ceeddd22d95553 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Thu, 28 Aug 2025 10:57:55 +0100 Subject: [PATCH 43/44] Refactor. --- .../Sources/Optimizer/Analysis/LoopTree.swift | 46 ++++++++++--------- .../ClosureSpecialization.swift | 3 +- .../LoopInvariantCodeMotion.swift | 42 +++++++++-------- 3 files changed, 50 insertions(+), 41 deletions(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift index b49f067ad4fa3..aab4605926c3b 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -80,32 +80,36 @@ struct Loop { return loopBlock.successors.contains { !contains(block: $0) } } - func getBlocksThatDominateAllExitingAndLatchBlocks(domTree: DominatorTree) -> some Sequence { - return getBlocksThatDominateAllExitingAndLatchBlocksHelper(bb: header, domTree: domTree) + func getBlocksThatDominateAllExitingAndLatchBlocks(_ context: FunctionPassContext) -> [BasicBlock] { + var result: [BasicBlock] = [] + var cachedExitingAndLatchBlocks = Stack(context) + var workList = BasicBlockWorklist(context) + defer { + cachedExitingAndLatchBlocks.deinitialize() + workList.deinitialize() + } + + cachedExitingAndLatchBlocks.append(contentsOf: exitingAndLatchBlocks) + workList.pushIfNotVisited(header) + + while let block = workList.pop() { + guard cachedExitingAndLatchBlocks.allSatisfy({ exitBlock in + return block.dominates(exitBlock, context.dominatorTree) + }) else { + continue + } + + result.append(block) + + workList.pushIfNotVisited(contentsOf: context.dominatorTree.getChildren(of: block)) + } + + return result } func contains(block: BasicBlock) -> Bool { return bridged.contains(block.bridged) } - - private func getBlocksThatDominateAllExitingAndLatchBlocksHelper( - bb: BasicBlock, - domTree: DominatorTree - ) -> some Sequence { - guard exitingAndLatchBlocks.allSatisfy({ exitBlock in - return bb.dominates(exitBlock, domTree) - }) else { - return [] - } - - return [bb] + domTree.getChildren(of: bb).lazy - .flatMap { child in - getBlocksThatDominateAllExitingAndLatchBlocksHelper( - bb: child, - domTree: domTree - ) - } - } func splitCriticalExitingAndBackEdges(_ context: FunctionPassContext) { for exitingOrLatchBlock in exitingAndLatchBlocks { diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift index 09f656f10d15e..be811c64ebf6d 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift @@ -247,9 +247,10 @@ private func getOrCreateSpecializedFunction( context.buildSpecializedFunction( specializedFunction: specializedFunction, buildFn: { (emptySpecializedFunction, functionPassContext) in - let closureSpecCloner = Cloner( + var closureSpecCloner = Cloner( cloneToEmptyFunction: emptySpecializedFunction, functionPassContext) closureSpecCloner.cloneAndSpecializeFunctionBody(using: pullbackClosureInfo) + closureSpecCloner.deinitialize() }) return (specializedFunction, false) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index 66330571bc0c0..b1f63ce8f687a 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// import SIL -import OptimizerBridging /// Hoist loop invariant code out of innermost loops. let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion") { function, context in @@ -65,7 +64,9 @@ private struct MovableInstructions { private struct AnalyzedInstructions { /// Side effects of the loop. var loopSideEffects: StackWithCount + private var blockSideEffectBottomMarker: StackWithCount.Marker + /// Side effects of the currently analyzed block. var sideEffectsOfCurrentBlock: StackWithCount.Segment { return StackWithCount.Segment( @@ -201,7 +202,7 @@ private func analyzeInstructions( ) { // Check against side-effects within the same block. // Side-effects in other blocks are checked later (after we - // scanned all blocks of the loop). + // scanned all blocks of the loop) in `collectHoistableGlobalInitCalls`. analyzedInstructions.globalInitCalls.append(applyInst) } @@ -244,11 +245,12 @@ private func collectHoistableReadOnlyApplies( var counter = 0 for readOnlyApply in analyzedInstructions.readOnlyApplies { // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. - if counter * analyzedInstructions.loopSideEffects.count < 8000, readOnlyApply.isSafeReadOnlyApply( - for: analyzedInstructions.loopSideEffects, - context.aliasAnalysis, - context.calleeAnalysis - ) { + if counter * analyzedInstructions.loopSideEffects.count < 8000, + readOnlyApply.isSafeReadOnlyApply( + for: analyzedInstructions.loopSideEffects, + context.aliasAnalysis, + context.calleeAnalysis + ) { movableInstructions.hoistUp.append(readOnlyApply) } counter += 1 @@ -264,11 +266,14 @@ private func collectHoistableGlobalInitCalls( ) { for globalInitCall in analyzedInstructions.globalInitCalls { // Check against side effects which are "before" (i.e. post-dominated by) the global initializer call. + // + // The effects in the same block have already been checked before + // adding this global init call to `analyzedInstructions.globalInitCalls` in `analyzeInstructions`. if globalInitCall.parentBlock.postDominates(loop.preheader!, context.postDominatorTree), !globalInitCall.globalInitMayConflictWith( - loopSideEffects: analyzedInstructions.loopSideEffects, - context.aliasAnalysis, - context.postDominatorTree + loopSideEffects: analyzedInstructions.loopSideEffects, + context.aliasAnalysis, + context.postDominatorTree ) { movableInstructions.hoistUp.append(globalInitCall) } @@ -619,7 +624,7 @@ private extension MovableInstructions { /// Only hoists instructions in blocks that dominate all exit and latch blocks. /// It doesn't hoist instructions speculatively. mutating func hoistInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { - let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(domTree: context.dominatorTree) + let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(context) var changed = false for bb in dominatingBlocks { @@ -633,7 +638,7 @@ private extension MovableInstructions { /// Sink instructions. mutating func sinkInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { - let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(domTree: context.dominatorTree) + let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(context) var changed = false for inst in sinkDown where dominatingBlocks.contains(inst.parentBlock) { @@ -681,13 +686,10 @@ private extension MovableInstructions { accessPath: AccessPath, context: FunctionPassContext ) -> Bool { - var changed = false - // Initially load the value in the loop pre header. let builder = Builder(before: loop.preheader!.terminator, context) var storeAddr: Value? - // Set all stored values as available values in the ssaUpdater. // If there are multiple stores in a block, only the last one counts. for case let storeInst as StoreInst in loadsAndStores where storeInst.isNonInitializingStoreTo(accessPath) { // If a store just stores the loaded value, bail. The operand (= the load) @@ -695,7 +697,7 @@ private extension MovableInstructions { // This corner case is surprisingly hard to handle, so we just give up. if let srcLoadInst = storeInst.source as? LoadInst, srcLoadInst.loadsFrom(accessPath) { - return changed + return false } if storeAddr == nil { @@ -705,12 +707,12 @@ private extension MovableInstructions { // must be interchangeable. It won't work if stores different types // because of casting or payload extraction even though they have the // same access path. - return changed + return false } } guard let storeAddr else { - return changed + return false } var ssaUpdater = SSAUpdater( @@ -720,6 +722,7 @@ private extension MovableInstructions { context ) + // Set all stored values as available values in the ssaUpdater. for case let storeInst as StoreInst in loadsAndStores where storeInst.isNonInitializingStoreTo(accessPath) { ssaUpdater.addAvailableValue(storeInst.source, in: storeInst.parentBlock) } @@ -743,7 +746,7 @@ private extension MovableInstructions { return .defaultValue } }) else { - return changed + return false } let ownership: LoadInst.LoadOwnership = loop.preheader!.terminator.parentFunction.hasOwnership ? .trivial : .unqualified @@ -751,6 +754,7 @@ private extension MovableInstructions { let initialLoad = builder.createLoad(fromAddress: initialAddr, ownership: ownership) ssaUpdater.addAvailableValue(initialLoad, in: loop.preheader!) + var changed = false var currentBlock: BasicBlock? var currentVal: Value? From fe047186d25b928acfc29a7e44d0e67be3dae0f5 Mon Sep 17 00:00:00 2001 From: Jakub Florek Date: Thu, 28 Aug 2025 11:50:04 +0100 Subject: [PATCH 44/44] Remove the call to .count(where: ) --- .../Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift index b1f63ce8f687a..d4e6340d7a2b8 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -531,7 +531,8 @@ private extension AnalyzedInstructions { continue } - if exitingBlocksSet.contains(block) && block.successors.count(where: { $0.terminator is UnreachableInst }) != block.successors.count { + if exitingBlocksSet.contains(block), + block.successors.filter({ $0.terminator is UnreachableInst }).count != block.successors.count { return false }