diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt index bedf3296a61df..4f95a146351ba 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt @@ -9,6 +9,7 @@ swift_compiler_sources(Optimizer AliasAnalysis.swift CalleeAnalysis.swift + LoopTree.swift DeadEndBlocksAnalysis.swift DominatorTree.swift PostDominatorTree.swift) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift index c7c106d879c08..132434c5eb028 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift @@ -15,6 +15,31 @@ import OptimizerBridging struct DominatorTree { let bridged: BridgedDomTree + + func getChildren(of block: BasicBlock) -> DomChildren { + return DomChildren(bridgedDomTree: bridged, bb: block) + } +} + +struct DomChildren: BridgedRandomAccessCollection { + private let bridgedDomTree: BridgedDomTree + private let block: BasicBlock + + let count: Int + + var startIndex: Int { return 0 } + var endIndex: Int { return count } + + init(bridgedDomTree: BridgedDomTree, bb: BasicBlock) { + self.bridgedDomTree = bridgedDomTree + self.block = bb + self.count = bridgedDomTree.getNumberOfChildren(bb.bridged) + } + + subscript(_ index: Int) -> BasicBlock { + assert(index >= startIndex && index < endIndex) + return bridgedDomTree.getChildAt(block.bridged, index).block + } } extension BasicBlock { diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift new file mode 100644 index 0000000000000..aab4605926c3b --- /dev/null +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -0,0 +1,213 @@ +//===--- LoopTree.swift ---------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SIL +import OptimizerBridging + +/// Describes top level loops. +struct LoopTree { + fileprivate let bridged: BridgedLoopTree + + let loops: TopLevelLoopArray + + init(bridged: BridgedLoopTree, context: FunctionPassContext) { + self.bridged = bridged + self.loops = TopLevelLoopArray(bridged) + } +} + +/// Describes a loop with its children. +struct Loop { + private let bridged: BridgedLoop + + let innerLoops: LoopArray + let loopBlocks: LoopBlocks + + var exitingAndLatchBlocks: some Sequence { + return header.predecessors.lazy + .filter { predecessor in + contains(block: predecessor) && !isLoopExiting(loopBlock: predecessor) + } + exitingBlocks + } + + /// Exit blocks of the loop. + /// + /// - Note: Some exit blocks will be duplicated if the loop has critical edges. + var exitBlocks: some Sequence { + return loopBlocks.lazy + .flatMap(\.successors) + .filter { !contains(block: $0) } + } + + var exitingBlocks: some Sequence { + return loopBlocks.lazy + .filter { isLoopExiting(loopBlock: $0) } + } + + init(bridged: BridgedLoop) { + self.bridged = bridged + self.innerLoops = LoopArray(bridged) + self.loopBlocks = LoopBlocks(bridged) + } + + var preheader: BasicBlock? { + bridged.getPreheader().block + } + + var header: BasicBlock { + bridged.getHeader().block + } + + /// Returns `true` if the loop has exactly one exit block. + var hasSingleExitBlock: Bool { + return exitBlocks.singleElement != nil + } + + var hasNoExitBlocks: Bool { + return exitBlocks.isEmpty + } + + private func isLoopExiting(loopBlock: BasicBlock) -> Bool { + return loopBlock.successors.contains { !contains(block: $0) } + } + + func getBlocksThatDominateAllExitingAndLatchBlocks(_ context: FunctionPassContext) -> [BasicBlock] { + var result: [BasicBlock] = [] + var cachedExitingAndLatchBlocks = Stack(context) + var workList = BasicBlockWorklist(context) + defer { + cachedExitingAndLatchBlocks.deinitialize() + workList.deinitialize() + } + + cachedExitingAndLatchBlocks.append(contentsOf: exitingAndLatchBlocks) + workList.pushIfNotVisited(header) + + while let block = workList.pop() { + guard cachedExitingAndLatchBlocks.allSatisfy({ exitBlock in + return block.dominates(exitBlock, context.dominatorTree) + }) else { + continue + } + + result.append(block) + + workList.pushIfNotVisited(contentsOf: context.dominatorTree.getChildren(of: block)) + } + + return result + } + + func contains(block: BasicBlock) -> Bool { + return bridged.contains(block.bridged) + } + + func splitCriticalExitingAndBackEdges(_ context: FunctionPassContext) { + for exitingOrLatchBlock in exitingAndLatchBlocks { + for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !contains(block: succesor) { + splitCriticalEdge( + from: exitingOrLatchBlock.terminator.parentBlock, + toEdgeIndex: index, + dominatorTree: context.dominatorTree, + loopTree: context.loopTree, + context + ) + } + } + } +} + +struct TopLevelLoopArray: BridgedRandomAccessCollection { + private let bridgedLoopTree: BridgedLoopTree + + let count: Int + + var startIndex: Int { return 0 } + var endIndex: Int { return count } + + init(_ bridgedLoopTree: BridgedLoopTree) { + self.bridgedLoopTree = bridgedLoopTree + self.count = bridgedLoopTree.getTopLevelLoopCount() + } + + subscript(_ index: Int) -> Loop { + assert(index >= startIndex && index < endIndex) + return Loop(bridged: bridgedLoopTree.getLoop(index)) + } +} + +struct LoopArray: BridgedRandomAccessCollection { + private let bridgedLoop: BridgedLoop + + let count: Int + + var startIndex: Int { return 0 } + var endIndex: Int { return count } + + init(_ bridgedLoop: BridgedLoop) { + self.bridgedLoop = bridgedLoop + self.count = bridgedLoop.getInnerLoopCount() + } + + subscript(_ index: Int) -> Loop { + assert(index >= startIndex && index < endIndex) + return Loop(bridged: bridgedLoop.getInnerLoop(index)) + } +} + +struct LoopBlocks: BridgedRandomAccessCollection { + private let bridgedLoop: BridgedLoop + + let count: Int + + var startIndex: Int { return 0 } + var endIndex: Int { return count } + + init(_ bridgedLoop: BridgedLoop) { + self.bridgedLoop = bridgedLoop + self.count = bridgedLoop.getBasicBlockCount() + } + + subscript(_ index: Int) -> BasicBlock { + assert(index >= startIndex && index < endIndex) + return bridgedLoop.getBasicBlock(index).block + } +} + +func splitEdge( + from block: BasicBlock, + toEdgeIndex: Int, + dominatorTree: DominatorTree, + loopTree: LoopTree, + _ context: some MutatingContext +) -> BasicBlock { + let result = loopTree.bridged.splitEdge(block.bridged, toEdgeIndex, dominatorTree.bridged).block + + context.notifyBranchesChanged() + return result +} + +/// If the specified edge is critical, the function returns inserted block. Otherwise returns `nil`. +@discardableResult +func splitCriticalEdge( + from block: BasicBlock, + toEdgeIndex: Int, + dominatorTree: DominatorTree, + loopTree: LoopTree, + _ context: some MutatingContext +) -> BasicBlock? { + guard block.isCriticalEdge(edgeIndex: toEdgeIndex) else { + return nil + } + + return splitEdge(from: block, toEdgeIndex: toEdgeIndex, dominatorTree: dominatorTree, loopTree: loopTree, context) +} diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift index 5d09c6d9a137f..0f99978d8d2b9 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift @@ -311,10 +311,9 @@ private struct FunctionSpecializations { // This can happen if a previous run of the pass already created this specialization. return } - let cloner = SpecializationCloner(emptySpecializedFunction: specializedFunc, context) - cloner.cloneFunctionBody(from: original) - context.buildSpecializedFunction(specializedFunction: specializedFunc) { (specializedFunc, specContext) in + cloneFunction(from: original, toEmpty: specializedFunc, specContext) + replaceBoxWithStackArguments(in: specializedFunc, original: original, specContext) } context.notifyNewFunction(function: specializedFunc, derivedFrom: original) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt index 52fe55235aee5..c0d7d0fada55e 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt @@ -25,6 +25,7 @@ swift_compiler_sources(Optimizer LifetimeDependenceDiagnostics.swift LifetimeDependenceInsertion.swift LifetimeDependenceScopeFixup.swift + LoopInvariantCodeMotion.swift ObjectOutliner.swift ObjCBridgingOptimization.swift MergeCondFails.swift diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift index fce5c35365d19..be811c64ebf6d 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift @@ -247,9 +247,10 @@ private func getOrCreateSpecializedFunction( context.buildSpecializedFunction( specializedFunction: specializedFunction, buildFn: { (emptySpecializedFunction, functionPassContext) in - let closureSpecCloner = SpecializationCloner( - emptySpecializedFunction: emptySpecializedFunction, functionPassContext) + var closureSpecCloner = Cloner( + cloneToEmptyFunction: emptySpecializedFunction, functionPassContext) closureSpecCloner.cloneAndSpecializeFunctionBody(using: pullbackClosureInfo) + closureSpecCloner.deinitialize() }) return (specializedFunction, false) @@ -731,15 +732,14 @@ private func markConvertedAndReabstractedClosuresAsUsed( } } -extension SpecializationCloner { +extension Cloner where Context == FunctionPassContext { fileprivate func cloneAndSpecializeFunctionBody(using pullbackClosureInfo: PullbackClosureInfo) { self.cloneEntryBlockArgsWithoutOrigClosures(usingOrigCalleeAt: pullbackClosureInfo) let (allSpecializedEntryBlockArgs, closureArgIndexToAllClonedReleasableClosures) = cloneAllClosures(at: pullbackClosureInfo) - self.cloneFunctionBody( - from: pullbackClosureInfo.pullbackFn, entryBlockArguments: allSpecializedEntryBlockArgs) + self.cloneFunctionBody(from: pullbackClosureInfo.pullbackFn, entryBlockArguments: allSpecializedEntryBlockArgs) self.insertCleanupCodeForClonedReleasableClosures( from: pullbackClosureInfo, @@ -750,8 +750,8 @@ extension SpecializationCloner { usingOrigCalleeAt pullbackClosureInfo: PullbackClosureInfo ) { let originalEntryBlock = pullbackClosureInfo.pullbackFn.entryBlock - let clonedFunction = self.cloned - let clonedEntryBlock = self.entryBlock + let clonedFunction = self.targetFunction + let clonedEntryBlock = self.getOrCreateEntryBlock() originalEntryBlock.arguments .enumerated() @@ -782,7 +782,8 @@ extension SpecializationCloner { ) { func entryBlockArgsWithOrigClosuresSkipped() -> [Value?] { - var clonedNonClosureEntryBlockArgs = self.entryBlock.arguments.makeIterator() + let clonedEntryBlock = self.getOrCreateEntryBlock() + var clonedNonClosureEntryBlockArgs = clonedEntryBlock.arguments.makeIterator() return pullbackClosureInfo.pullbackFn .entryBlock @@ -823,8 +824,8 @@ extension SpecializationCloner { { let (origToClonedValueMap, capturedArgRange) = self.addEntryBlockArgs( forValuesCapturedBy: closureArgDesc) - let clonedFunction = self.cloned - let clonedEntryBlock = self.entryBlock + let clonedFunction = self.targetFunction + let clonedEntryBlock = self.getOrCreateEntryBlock() let clonedClosureArgs = Array(clonedEntryBlock.arguments[capturedArgRange]) let builder = @@ -853,8 +854,8 @@ extension SpecializationCloner { -> (origToClonedValueMap: [HashableValue: Value], capturedArgRange: Range) { var origToClonedValueMap: [HashableValue: Value] = [:] - let clonedFunction = self.cloned - let clonedEntryBlock = self.entryBlock + let clonedFunction = self.targetFunction + let clonedEntryBlock = self.getOrCreateEntryBlock() let capturedArgRangeStart = clonedEntryBlock.arguments.count @@ -908,8 +909,8 @@ extension SpecializationCloner { } } - if self.context.needFixStackNesting { - self.context.fixStackNesting(in: self.cloned) + if (self.context.needFixStackNesting) { + self.context.fixStackNesting(in: targetFunction) } } } @@ -1425,12 +1426,10 @@ private struct PullbackClosureInfo { } func specializedCalleeName(_ context: FunctionPassContext) -> String { - let closureArgs = Array(self.closureArgDescriptors.map { $0.closure }) - let closureIndices = Array(self.closureArgDescriptors.map { $0.closureArgIndex }) - - return context.mangle( - withClosureArguments: closureArgs, closureArgIndices: closureIndices, - from: pullbackFn) + let closureArgs = Array(self.closureArgDescriptors.map { + (argumentIndex: $0.closureArgIndex, argumentValue: $0.closure) + }) + return context.mangle(withClosureArguments: closureArgs, from: pullbackFn) } } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift index 3d862fff6d2f1..6f181f1339d95 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift @@ -271,7 +271,7 @@ private indirect enum GlobalInitValue { /// Creates SIL for this global init value in the initializer of the `global`. func materialize(into global: GlobalVariable, from function: Function, _ context: FunctionPassContext) { - var cloner = StaticInitCloner(cloneTo: global, context) + var cloner = Cloner(cloneToGlobal: global, context) defer { cloner.deinitialize() } let builder = Builder(staticInitializerOf: global, context) @@ -280,7 +280,7 @@ private indirect enum GlobalInitValue { private func materializeRecursively( type: Type, - _ cloner: inout StaticInitCloner, + _ cloner: inout Cloner, _ builder: Builder, _ function: Function ) -> Value { @@ -289,7 +289,7 @@ private indirect enum GlobalInitValue { fatalError("cannot materialize undefined init value") case .constant(let value): - return cloner.clone(value) + return cloner.cloneRecursivelyToGlobal(value: value) case .aggregate(let fields): if type.isStruct { diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift new file mode 100644 index 0000000000000..c250890da4080 --- /dev/null +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -0,0 +1,1129 @@ +//===--- LoopInvariantCodeMotion.swift ------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SIL + +/// Hoist loop invariant code out of innermost loops. +let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion") { function, context in + for loop in context.loopTree.loops { + optimizeTopLevelLoop(topLevelLoop: loop, context) + } +} + +private func optimizeTopLevelLoop(topLevelLoop: Loop, _ context: FunctionPassContext) { + var innerLoops = Stack(context) + defer { innerLoops.deinitialize() } + + getWorkList(forLoop: topLevelLoop, workList: &innerLoops) + + while let thisLoop = innerLoops.pop() { + // We only support Loops with a preheader. + guard thisLoop.preheader != nil else { + continue + } + + var thisLoopChanged = false + + repeat { + var movableInstructions = analyzeLoopAndSplitLoads(loop: thisLoop, context) + thisLoopChanged = optimizeLoop(loop: thisLoop, movableInstructions: &movableInstructions, context) + } while thisLoopChanged + } +} + +/// Creates post-order DFS work list of inner loops. +private func getWorkList(forLoop loop: Loop, workList: inout Stack) { + workList.push(loop) + + for innerLoop in loop.innerLoops { + getWorkList(forLoop: innerLoop, workList: &workList) + } +} + +/// Instructions that can be moved outside the loop. +private struct MovableInstructions { + var loadAndStoreAccessPaths: [AccessPath] = [] + + var speculativelyHoistable: [Instruction] = [] + var loadsAndStores: [Instruction] = [] + var hoistUp: [Instruction] = [] + var sinkDown: [Instruction] = [] + var scopedInsts: [Instruction] = [] +} + +/// Analyzed instructions inside the currently processed loop. +private struct AnalyzedInstructions { + /// Side effects of the loop. + var loopSideEffects: StackWithCount + + private var blockSideEffectBottomMarker: StackWithCount.Marker + + /// Side effects of the currently analyzed block. + var sideEffectsOfCurrentBlock: StackWithCount.Segment { + return StackWithCount.Segment( + in: loopSideEffects, + low: blockSideEffectBottomMarker, + high: loopSideEffects.top + ) + } + + /// Contains either: + /// * an apply to the addressor of the global + /// * a builtin "once" of the global initializer + var globalInitCalls: Stack + var readOnlyApplies: Stack + var loads: Stack + var stores: Stack + var beginAccesses: Stack + var fullApplies: Stack + + /// `true` if the loop has instructions which (may) read from memory, which are not in `Loads` and not in `sideEffects`. + var hasOtherMemReadingInsts = false + + /// `true` if one of the side effects might release. + lazy var sideEffectsMayRelease = loopSideEffects.contains(where: { $0.mayRelease }) + + init (_ context: FunctionPassContext) { + self.loopSideEffects = StackWithCount(context) + self.blockSideEffectBottomMarker = loopSideEffects.top + + self.globalInitCalls = Stack(context) + self.readOnlyApplies = Stack(context) + self.loads = Stack(context) + self.stores = Stack(context) + self.beginAccesses = Stack(context) + self.fullApplies = Stack(context) + } + + mutating func deinitialize() { + readOnlyApplies.deinitialize() + globalInitCalls.deinitialize() + loopSideEffects.deinitialize() + loads.deinitialize() + stores.deinitialize() + beginAccesses.deinitialize() + fullApplies.deinitialize() + } + + /// Mark the start of currently processed block side effects. + mutating func markBeginOfBlock() { + blockSideEffectBottomMarker = loopSideEffects.top + } +} + +/// Analyzes `loop` for hosting/sinking potential. +/// Computes `MovableInstructions` we may be able to move out of the loop +/// +/// This may split some loads into smaller loads. +private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext) -> MovableInstructions { + // TODO: Remove once uses lowered OSSA. + loop.splitCriticalExitingAndBackEdges(context) + + var movableInstructions = MovableInstructions() + var analyzedInstructions = AnalyzedInstructions(context) + defer { analyzedInstructions.deinitialize() } + + analyzeInstructions(in: loop, &analyzedInstructions, &movableInstructions, context) + + collectHoistableReadOnlyApplies(analyzedInstructions, &movableInstructions, context) + + collectHoistableGlobalInitCalls(in: loop, analyzedInstructions, &movableInstructions, context) + + collectProjectableAccessPathsAndSplitLoads(in: loop, &analyzedInstructions, &movableInstructions, context) + + collectMovableInstructions(in: loop, &analyzedInstructions, &movableInstructions, context) + + return movableInstructions +} + +/// Analyze instructions inside the `loop`. Compute side effects and populate `analyzedInstructions`. +/// +/// - note: Ideally, `movableInstructions` should be fully computed in `collectMovableInstructions`. +private func analyzeInstructions( + in loop: Loop, + _ analyzedInstructions: inout AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + for bb in loop.loopBlocks { + analyzedInstructions.markBeginOfBlock() + + for inst in bb.instructions { + // TODO: Remove once support for values with ownership implemented. + if inst.hasOwnershipOperandsOrResults { + analyzedInstructions.analyzeSideEffects(ofInst: inst) + + // Collect fullApplies to be checked in analyzeBeginAccess + if let fullApply = inst as? FullApplySite { + analyzedInstructions.fullApplies.append(fullApply) + } + + continue + } + + switch inst { + case is FixLifetimeInst: + break // We can ignore the side effects of FixLifetimes + case let loadInst as LoadInst: + analyzedInstructions.loads.append(loadInst) + case let storeInst as StoreInst: + switch storeInst.storeOwnership { + case .assign, .initialize: + continue // TODO: Add support + case .unqualified, .trivial: + break + } + analyzedInstructions.stores.append(storeInst) + analyzedInstructions.analyzeSideEffects(ofInst: storeInst) + case let beginAccessInst as BeginAccessInst: + analyzedInstructions.beginAccesses.append(beginAccessInst) + analyzedInstructions.analyzeSideEffects(ofInst: beginAccessInst) + case let refElementAddrInst as RefElementAddrInst: + movableInstructions.speculativelyHoistable.append(refElementAddrInst) + case let condFailInst as CondFailInst: + analyzedInstructions.analyzeSideEffects(ofInst: condFailInst) + case let applyInst as ApplyInst: + if applyInst.isSafeReadOnlyApply(context.calleeAnalysis) { + analyzedInstructions.readOnlyApplies.append(applyInst) + } else if let callee = applyInst.referencedFunction, + callee.isGlobalInitFunction, // Calls to global inits are different because we don't care about side effects which are "after" the call in the loop. + !applyInst.globalInitMayConflictWith( + blockSideEffectSegment: analyzedInstructions.sideEffectsOfCurrentBlock, + context.aliasAnalysis + ) { + // Check against side-effects within the same block. + // Side-effects in other blocks are checked later (after we + // scanned all blocks of the loop) in `collectHoistableGlobalInitCalls`. + analyzedInstructions.globalInitCalls.append(applyInst) + } + + // Check for array semantics and side effects - same as default + fallthrough + default: + switch inst { + case let fullApply as FullApplySite: + analyzedInstructions.fullApplies.append(fullApply) + case let builtinInst as BuiltinInst: + switch builtinInst.id { + case .Once, .OnceWithContext: + if !builtinInst.globalInitMayConflictWith( + blockSideEffectSegment: analyzedInstructions.sideEffectsOfCurrentBlock, + context.aliasAnalysis + ) { + analyzedInstructions.globalInitCalls.append(builtinInst) + } + default: break + } + default: break + } + + analyzedInstructions.analyzeSideEffects(ofInst: inst) + + if inst.canBeHoisted(outOf: loop, context) { + movableInstructions.hoistUp.append(inst) + } + } + } + } +} + +/// Process collected read only applies. Moves them to `hoistUp` if they don't conflict with any side effects. +private func collectHoistableReadOnlyApplies( + _ analyzedInstructions: AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + var counter = 0 + for readOnlyApply in analyzedInstructions.readOnlyApplies { + // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. + if counter * analyzedInstructions.loopSideEffects.count < 8000, + readOnlyApply.isSafeReadOnlyApply( + for: analyzedInstructions.loopSideEffects, + context.aliasAnalysis, + context.calleeAnalysis + ) { + movableInstructions.hoistUp.append(readOnlyApply) + } + counter += 1 + } +} + +/// Process collected global init calls. Moves them to `hoistUp` if they don't conflict with any side effects. +private func collectHoistableGlobalInitCalls( + in loop: Loop, + _ analyzedInstructions: AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + for globalInitCall in analyzedInstructions.globalInitCalls { + // Check against side effects which are "before" (i.e. post-dominated by) the global initializer call. + // + // The effects in the same block have already been checked before + // adding this global init call to `analyzedInstructions.globalInitCalls` in `analyzeInstructions`. + if globalInitCall.parentBlock.postDominates(loop.preheader!, context.postDominatorTree), + !globalInitCall.globalInitMayConflictWith( + loopSideEffects: analyzedInstructions.loopSideEffects, + context.aliasAnalysis, + context.postDominatorTree + ) { + movableInstructions.hoistUp.append(globalInitCall) + } + } +} + +/// Collect memory locations for which we can move all loads and stores out of the loop. +/// `loads` may mutate during this loop. +private func collectProjectableAccessPathsAndSplitLoads( + in loop: Loop, + _ analyzedInstructions: inout AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + if !analyzedInstructions.hasOtherMemReadingInsts { + for storeInst in analyzedInstructions.stores { + let accessPath = storeInst.destination.accessPath + if accessPath.isLoopInvariant(loop: loop), + analyzedInstructions.isOnlyLoadedAndStored( + accessPath: accessPath, + storeAddr: storeInst.destination, + context.aliasAnalysis + ), + !movableInstructions.loadAndStoreAccessPaths.contains(accessPath), + // This is not a requirement for functional correctness, but we don't want to + // _speculatively_ load and store the value (outside of the loop). + analyzedInstructions.storesCommonlyDominateExits(of: loop, storingTo: accessPath, context), + analyzedInstructions.splitLoads( + storeAddr: storeInst.destination, + accessPath: accessPath, + context + ) { + movableInstructions.loadAndStoreAccessPaths.append(accessPath) + } + } + } +} + +/// Computes movable instructions using computed analyzed instructions. +private func collectMovableInstructions( + in loop: Loop, + _ analyzedInstructions: inout AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + var loadInstCounter = 0 + for bb in loop.loopBlocks { + for inst in bb.instructions { + // TODO: Remove once support for values with ownership implemented. + if inst.hasOwnershipOperandsOrResults { + continue + } + + switch inst { + case let fixLifetimeInst as FixLifetimeInst: + guard fixLifetimeInst.parentBlock.dominates(loop.preheader!, context.dominatorTree) else { + continue + } + + if !analyzedInstructions.sideEffectsMayRelease || !analyzedInstructions.loopSideEffectsMayWriteTo(address: fixLifetimeInst.operand.value, context.aliasAnalysis) { + movableInstructions.sinkDown.append(fixLifetimeInst) + } + case let loadInst as LoadInst: + // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. + if loadInstCounter * analyzedInstructions.loopSideEffects.count < 8000, + !analyzedInstructions.loopSideEffectsMayWriteTo(address: loadInst.operand.value, context.aliasAnalysis), + loadInst.loadOwnership != .take, loadInst.loadOwnership != .copy { // TODO: Add support for take and copy loads. + movableInstructions.hoistUp.append(loadInst) + } + + loadInstCounter += 1 + + movableInstructions.loadsAndStores.append(loadInst) + case let storeInst as StoreInst: + switch storeInst.storeOwnership { + case .assign, .initialize: + continue // TODO: Add support + case .unqualified, .trivial: + break + } + movableInstructions.loadsAndStores.append(storeInst) + case let condFailInst as CondFailInst: + // We can (and must) hoist cond_fail instructions if the operand is + // invariant. We must hoist them so that we preserve memory safety. A + // cond_fail that would have protected (executed before) a memory access + // must - after hoisting - also be executed before said access. + movableInstructions.hoistUp.append(condFailInst) + case let beginAccessInst as BeginAccessInst: + if beginAccessInst.canBeHoisted(outOf: loop, analyzedInstructions: analyzedInstructions, context) { + movableInstructions.scopedInsts.append(beginAccessInst) + } + default: + break + } + } + } +} + +/// Optimizes the loop by performing in following order: +/// - speculative hoist +/// - projection, hoist and sink of loads and stores +/// - hoist of instructions that are guaranteed to be executed +/// - sink +/// - hoist with sink of scoped instructions +private func optimizeLoop( + loop: Loop, + movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) -> Bool { + var changed = false + + // TODO: If we hoist tuple_element_addr and struct_element_addr instructions here, hoistAndSinkLoadsAndStores could converge after just one execution! + changed = movableInstructions.speculativelyHoistInstructions(outOf: loop, context) || changed + changed = movableInstructions.hoistAndSinkLoadsAndStores(outOf: loop, context) || changed + changed = movableInstructions.hoistInstructions(outOf: loop, context) || changed + changed = movableInstructions.sinkInstructions(outOf: loop, context) || changed + changed = movableInstructions.hoistWithSinkScopedInstructions(outOf: loop, context) || changed + + return changed +} + +extension BasicBlock { + func containsStoresTo(accessPath: AccessPath) -> Bool { + return instructions.contains { inst in + return inst.operands.contains { operand in + if let storeInst = operand.instruction as? StoreInst, + storeInst.destination.accessPath == accessPath { + return true + } else { + return false + } + } + } + } +} + +private extension AnalyzedInstructions { + /// Adds side effects of `inst` to the analyzed instructions. + mutating func analyzeSideEffects(ofInst inst: Instruction) { + if inst.mayHaveSideEffects { + loopSideEffects.append(inst) + } else if inst.mayReadFromMemory { + hasOtherMemReadingInsts = true + } + } + + /// Returns true if all instructions in `sideEffects` which may alias with + /// this path are either loads or stores from this path. + /// + /// `storeAddr` is only needed for AliasAnalysis until we have an interface + /// that supports `AccessPath`. + func isOnlyLoadedAndStored( + accessPath: AccessPath, + storeAddr: Value, + _ aliasAnalysis: AliasAnalysis + ) -> Bool { + if (loopSideEffects.contains { sideEffect in + switch sideEffect { + case let storeInst as StoreInst: + if storeInst.isNonInitializingStoreTo(accessPath) { + return false + } + case let loadInst as LoadInst: + if loadInst.loadsFrom(accessPath) { + return false + } + default: break + } + + // Pass the original address value until we can fix alias analysis. + return sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) + }) { + return false + } + + if (loads.contains { loadInst in + loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) && loadInst.loadOwnership != .take && !loadInst.overlaps(accessPath: accessPath) + }) { + return false + } + + if (stores.contains { storeInst in + storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && !storeInst.isNonInitializingStoreTo(accessPath) + }) { + return false + } + + return true + } + + /// Returns `true` if all stores to `accessPath` commonly dominate the loop exits. + func storesCommonlyDominateExits(of loop: Loop, storingTo accessPath: AccessPath, _ context: FunctionPassContext) -> Bool { + var exitingBlocksSet = BasicBlockSet(context) + var storeBlocks = BasicBlockSet(context) + var worklist = BasicBlockWorklist(context) + defer { + exitingBlocksSet.deinitialize() + storeBlocks.deinitialize() + worklist.deinitialize() + } + + // Also a store in the pre-header dominates all exists. Although the situation + // is a bit different here: the store in the pre-header remains - it's not + // (re)moved by the LICM transformation. + // But even if the loop-stores are not dominating the loop exits, it + // makes sense to move them out of the loop if this case. When this is done, + // dead-store-elimination can then most likely eliminate the store in the + // pre-header. + // + // pre_header: + // store %v1 to %addr + // header: + // cond_br %cond, then, tail + // then: + // store %v2 to %addr // a conditional store in the loop + // br tail + // tail: + // cond_br %loop_cond, header, exit + // exit: + // + // will be transformed to + // + // pre_header: + // store %v1 to %addr // <- can be removed by DSE afterwards + // header: + // cond_br %cond, then, tail + // then: + // br tail + // tail(%phi): + // cond_br %loop_cond, header, exit + // exit: + // store %phi to %addr + // + if loop.preheader!.containsStoresTo(accessPath: accessPath) { + return true + } + + // Create a set of exiting blocks for efficient lookup later. + exitingBlocksSet.insert(contentsOf: loop.exitingBlocks) + + // Collect as many recognizable store parent blocks as possible. It's ok if not all stores are collected. + storeBlocks.insert(contentsOf: stores + .filter({ $0.destination.accessPath == accessPath }) + .map(\.parentBlock)) + + // If a store is in the loop header, we already know that it's dominating all loop exits. + if storeBlocks.contains(loop.header) { + return true + } + + // Starting from the header, check whether all stores are alive. + worklist.pushIfNotVisited(loop.header) + while let block = worklist.pop() { + if storeBlocks.contains(block) { + continue + } + + if exitingBlocksSet.contains(block), + block.successors.filter({ $0.terminator is UnreachableInst }).count != block.successors.count { + return false + } + + worklist.pushIfNotVisited(contentsOf: block.successors) + } + + return true + } + + /// Returns true if `loopSideEffects` contains any memory writes which + /// may alias with the memory `address`. + func loopSideEffectsMayWriteTo(address: Value, _ aliasAnalysis: AliasAnalysis) -> Bool { + return loopSideEffects + .contains { sideEffect in + sideEffect.mayWrite(toAddress: address, aliasAnalysis) + } + } + + /// Find all loads that contain `accessPath`. Split them into a load with + /// identical `accessPath` and a set of non-overlapping loads. Add the new + /// non-overlapping loads to `loads`. + mutating func splitLoads( + storeAddr: Value, + accessPath: AccessPath, + _ context: FunctionPassContext + ) -> Bool { + var newLoads = Stack(context) + defer { + loads.append(contentsOf: newLoads) + newLoads.deinitialize() + } + var splitCounter = 0 + + while let loadInst = loads.pop() { + // Found a load wider than the store to accessPath. + // + // SplitLoads is called for each unique access path in the loop that is + // only loaded from and stored to and this loop takes time proportional to: + // num-wide-loads x num-fields x num-loop-memops + // + // For each load wider than the store, it creates a new load for each field + // in that type. Each new load is inserted in the LoadsAndStores vector. To + // avoid super-linear behavior for large types (e.g. giant tuples), limit + // growth of new loads to an arbitrary constant factor per access path. + guard splitCounter <= 6 else { + newLoads.push(loadInst) + return false + } + + guard !loadInst.isDeleted, loadInst.operand.value.accessPath.contains(accessPath) else { + newLoads.push(loadInst) + continue + } + + guard let splitLoads = loadInst.trySplit(alongPath: accessPath.projectionPath, context) else { + newLoads.push(loadInst) + return false + } + + splitCounter += splitLoads.count + newLoads.append(contentsOf: splitLoads) + } + + return true + } +} + +private extension MovableInstructions { + /// Hoist instructions speculatively. + /// + /// Contrary to `hoistInstructions`, it doesn't only go through instructions in blocks that dominate all exits. + mutating func speculativelyHoistInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + var changed = false + + for inst in speculativelyHoistable { + changed = inst.hoist(outOf: loop, context) || changed + } + + return changed + } + + /// Hoists and sinks loads with matching stores. Projects loads. + mutating func hoistAndSinkLoadsAndStores(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + var changed = false + + for accessPath in loadAndStoreAccessPaths { + changed = hoistAndSinkLoadAndStore(outOf: loop, accessPath: accessPath, context: context) || changed + } + + return changed + } + + /// Only hoists instructions in blocks that dominate all exit and latch blocks. + /// It doesn't hoist instructions speculatively. + mutating func hoistInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(context) + var changed = false + + for bb in dominatingBlocks { + for inst in bb.instructions where hoistUp.contains(inst) { + changed = inst.hoist(outOf: loop, context) || changed + } + } + + return changed + } + + /// Sink instructions. + mutating func sinkInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(context) + var changed = false + + for inst in sinkDown where dominatingBlocks.contains(inst.parentBlock) { + changed = inst.sink(outOf: loop, context) || changed + } + + return changed + } + + /// Hoist and sink scoped instructions. + mutating func hoistWithSinkScopedInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + guard !loop.hasNoExitBlocks else { + return false + } + + var changed = false + + for specialInst in scopedInsts { + guard let beginAccessInst = specialInst as? BeginAccessInst else { + continue + } + + guard specialInst.hoist(outOf: loop, context) else { + continue + } + + // We only want to sink the first end_access and erase the rest to not introduce duplicates. + var sankFirst = false + for endAccess in beginAccessInst.endAccessInstructions { + if sankFirst { + context.erase(instruction: endAccess) + } else { + sankFirst = endAccess.sink(outOf: loop, context) + } + } + + changed = true + } + + return changed + } + + private mutating func hoistAndSinkLoadAndStore( + outOf loop: Loop, + accessPath: AccessPath, + context: FunctionPassContext + ) -> Bool { + // Initially load the value in the loop pre header. + let builder = Builder(before: loop.preheader!.terminator, context) + var storeAddr: Value? + + // If there are multiple stores in a block, only the last one counts. + for case let storeInst as StoreInst in loadsAndStores where storeInst.isNonInitializingStoreTo(accessPath) { + // If a store just stores the loaded value, bail. The operand (= the load) + // will be removed later, so it cannot be used as available value. + // This corner case is surprisingly hard to handle, so we just give up. + if let srcLoadInst = storeInst.source as? LoadInst, + srcLoadInst.loadsFrom(accessPath) { + return false + } + + if storeAddr == nil { + storeAddr = storeInst.destination + } else if storeInst.destination.type != storeAddr!.type { + // This transformation assumes that the values of all stores in the loop + // must be interchangeable. It won't work if stores different types + // because of casting or payload extraction even though they have the + // same access path. + return false + } + } + + guard let storeAddr else { + return false + } + + var ssaUpdater = SSAUpdater( + function: storeAddr.parentFunction, + type: storeAddr.type.objectType, + ownership: .none, + context + ) + + // Set all stored values as available values in the ssaUpdater. + for case let storeInst as StoreInst in loadsAndStores where storeInst.isNonInitializingStoreTo(accessPath) { + ssaUpdater.addAvailableValue(storeInst.source, in: storeInst.parentBlock) + } + + var cloner = Cloner(cloneBefore: loop.preheader!.terminator, context) + defer { cloner.deinitialize() } + + guard let initialAddr = (cloner.cloneRecursively(value: storeAddr) { srcAddr, cloner in + switch srcAddr { + case is AllocStackInst, is BeginBorrowInst, is MarkDependenceInst: + return .stopCloning + default: break + } + + // Clone projections until the address dominates preheader. + if srcAddr.parentBlock.dominates(loop.preheader!, context.dominatorTree) { + cloner.recordFoldedValue(srcAddr, mappedTo: srcAddr) + return .customValue(srcAddr) + } else { + // Return nil invalid to continue cloning. + return .defaultValue + } + }) else { + return false + } + + let ownership: LoadInst.LoadOwnership = loop.preheader!.terminator.parentFunction.hasOwnership ? .trivial : .unqualified + + let initialLoad = builder.createLoad(fromAddress: initialAddr, ownership: ownership) + ssaUpdater.addAvailableValue(initialLoad, in: loop.preheader!) + + var changed = false + var currentBlock: BasicBlock? + var currentVal: Value? + + // Remove all stores and replace the loads with the current value. + // + // This loop depends on loadsAndStores being in order the instructions appear in blocks. + for inst in loadsAndStores { + let block = inst.parentBlock + + if block != currentBlock { + currentBlock = block + currentVal = nil + } + + if let storeInst = inst as? StoreInst, storeInst.isNonInitializingStoreTo(accessPath) { + currentVal = storeInst.source + context.erase(instruction: storeInst) + changed = true + continue + } + + guard let loadInst = inst as? LoadInst, + loadInst.loadsFrom(accessPath) else { + continue + } + + // If we didn't see a store in this block yet, get the current value from the ssaUpdater. + let rootVal = currentVal ?? ssaUpdater.getValue(inMiddleOf: block) + + if loadInst.operand.value.accessPath == accessPath { + loadInst.replace(with: rootVal, context) + changed = true + continue + } + + guard let projectionPath = accessPath.getProjection(to: loadInst.operand.value.accessPath) else { + continue + } + + let builder = Builder(before: loadInst, context) + let projection = rootVal.createProjection(path: projectionPath, builder: builder) + loadInst.replace(with: projection, context) + + changed = true + } + + loadsAndStores.removeAll(where: { $0.isDeleted }) + + // Store back the value at all loop exits. + for exitBlock in loop.exitBlocks { + assert(exitBlock.hasSinglePredecessor, "Exiting edge should not be critical.") + + let builder = Builder(before: exitBlock.instructions.first!, context) + + let ownership: StoreInst.StoreOwnership = exitBlock.instructions.first!.parentFunction.hasOwnership ? .trivial : .unqualified + builder.createStore( + source: ssaUpdater.getValue(inMiddleOf: exitBlock), + destination: initialAddr, + ownership: ownership + ) + changed = true + } + + // In case the value is only stored but never loaded in the loop. + if initialLoad.uses.isEmpty { + context.erase(instruction: initialLoad) + } + + return changed + } +} + +private extension Instruction { + var hasOwnershipOperandsOrResults: Bool { + guard parentFunction.hasOwnership else { return false } + + return results.contains(where: { $0.ownership != .none }) + || operands.contains(where: { $0.value.ownership != .none }) + } + + /// Returns `true` if this instruction follows the default hoisting heuristic which means it + /// is not a terminator, allocation or deallocation and either a hoistable array semantics call or doesn't have memory effects. + func canBeHoisted(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + switch self { + case is TermInst, is Allocation, is Deallocation: + return false + case is ApplyInst: + switch arraySemanticsCallKind { + case .getCount, .getCapacity: + if canHoistArraySemanticsCall(to: loop.preheader!.terminator, context) { + return true + } + case .arrayPropsIsNativeTypeChecked: + return false + default: + break + } + default: + break + } + + if memoryEffects == .noEffects { + return true + } + + return false + } + + func hoist(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + guard operands.allSatisfy({ !loop.contains(block: $0.value.parentBlock) }) else { + return false + } + + let terminator = loop.preheader!.terminator + if canHoistArraySemanticsCall(to: terminator, context) { + hoistArraySemanticsCall(before: terminator, context) + } else { + move(before: terminator, context) + } + + if let singleValueInst = self as? SingleValueInstruction, + !(self is BeginAccessInst), + let identicalInst = (loop.preheader!.instructions.first { otherInst in + return singleValueInst != otherInst && singleValueInst.isIdenticalTo(otherInst) + }) { + guard let identicalSingleValueInst = identicalInst as? SingleValueInstruction else { + return true + } + + singleValueInst.replace(with: identicalSingleValueInst, context) + } + + return true + } + + func sink(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + let exitBlocks = loop.exitBlocks + var changed = false + + for exitBlock in exitBlocks { + assert(exitBlock.hasSinglePredecessor, "Exiting edge should not be critical.") + + if changed { + copy(before: exitBlock.instructions.first!, context) + } else { + move(before: exitBlock.instructions.first!, context) + changed = true + } + } + + return changed + } + + /// Returns `true` if `sideEffect` cannot be reordered with a call to this + /// global initializer. + private func globalInitMayConflictWith( + sideEffect: Instruction, + _ aliasAnalysis: AliasAnalysis + ) -> Bool { + switch sideEffect { + case let storeInst as StoreInst: + return mayReadOrWrite(address: storeInst.destinationOperand.value, aliasAnalysis) + case let loadInst as LoadInst: + return mayWrite(toAddress: loadInst.operand.value, aliasAnalysis) + case is CondFailInst: + return false + default: + return true + } + } + + /// Returns `true` if any of the instructions in `sideEffects` cannot be + /// reordered with a call to this global initializer (which is in the same basic + /// block). + func globalInitMayConflictWith( + blockSideEffectSegment: StackWithCount.Segment, + _ aliasAnalysis: AliasAnalysis + ) -> Bool { + return blockSideEffectSegment + .contains { sideEffect in + globalInitMayConflictWith( + sideEffect: sideEffect, + aliasAnalysis + ) + } + } + + /// Returns `true` if any of the instructions in `loopSideEffects` which are + /// post-dominated by a call to this global initializer cannot be reordered with + /// the call. + func globalInitMayConflictWith( + loopSideEffects: StackWithCount, + _ aliasAnalysis: AliasAnalysis, + _ postDomTree: PostDominatorTree + ) -> Bool { + return loopSideEffects + .contains { sideEffect in + // Only check instructions in blocks which are "before" (i.e. post-dominated + // by) the block which contains the init-call. + // Instructions which are before the call in the same block have already + // been checked. + parentBlock.strictlyPostDominates(sideEffect.parentBlock, postDomTree) && + globalInitMayConflictWith(sideEffect: sideEffect, aliasAnalysis) + } + } +} + +private extension StoreInst { + /// Returns a `true` if this store is a store to `accessPath`. + func isNonInitializingStoreTo(_ accessPath: AccessPath) -> Bool { + if self.storeOwnership == .initialize { + return false + } + + return accessPath == self.destination.accessPath + } +} + +private extension LoadInst { + /// Returns `true` if this load instruction loads from `accessPath` or a + /// projected address from `accessPath`. + func loadsFrom(_ accessPath: AccessPath) -> Bool { + return accessPath.getProjection(to: self.address.accessPath)?.isMaterializable ?? false + } + + func overlaps(accessPath: AccessPath) -> Bool { + // Don't use `AccessPath.mayOverlap`. We only want definite overlap. + return accessPath.isEqualOrContains(self.operand.value.accessPath) || self.operand.value.accessPath.isEqualOrContains(accessPath) + } +} + +private extension ApplyInst { + /// Returns `true` if this apply inst could be safely hoisted. + func isSafeReadOnlyApply(_ calleeAnalysis: CalleeAnalysis) -> Bool { + guard functionConvention.results.allSatisfy({ $0.convention == .unowned }) else { + return false + } + + if let callee = referencedFunction, + callee.hasSemanticsAttribute("array.props.isNativeTypeChecked") { + return false + } + + return !calleeAnalysis.getSideEffects(ofApply: self).memory.write + } + + /// Returns `true` if the `sideEffects` contain any memory writes which + /// may alias with any memory which is read by this `ApplyInst`. + /// - Note: This function should only be called on a read-only apply! + func isSafeReadOnlyApply( + for sideEffects: StackWithCount, + _ aliasAnalysis: AliasAnalysis, + _ calleeAnalysis: CalleeAnalysis + ) -> Bool { + if calleeAnalysis.getSideEffects(ofApply: self).memory == .noEffects { + return true + } + + // Check if the memory addressed by the argument may alias any writes. + for sideEffect in sideEffects { + switch sideEffect { + case let storeInst as StoreInst: + if storeInst.storeOwnership == .assign || + mayRead(fromAddress: storeInst.destination, aliasAnalysis) { + return false + } + case let copyAddrInst as CopyAddrInst: + if !copyAddrInst.isInitializationOfDestination || + mayRead(fromAddress: copyAddrInst.destination, aliasAnalysis) { + return false + } + case is ApplyInst, is BeginApplyInst, is TryApplyInst: + if calleeAnalysis.getSideEffects(ofApply: self).memory.write { + return false + } + case is CondFailInst, is StrongRetainInst, is UnmanagedRetainValueInst, + is RetainValueInst, is StrongRetainUnownedInst, is FixLifetimeInst, + is KeyPathInst, is DeallocStackInst, is DeallocStackRefInst, + is DeallocRefInst: + break + default: + if sideEffect.mayWriteToMemory { + return false + } + } + } + + return true + } +} + +private extension BeginAccessInst { + /// Returns `true` if this begin access is safe to hoist. + func canBeHoisted( + outOf loop: Loop, + analyzedInstructions: AnalyzedInstructions, + _ context: FunctionPassContext + ) -> Bool { + guard endAccessInstructions.allSatisfy({ loop.contains(block: $0.parentBlock)}) else { + return false + } + + let areBeginAccessesSafe = analyzedInstructions.beginAccesses + .allSatisfy { otherBeginAccessInst in + guard self != otherBeginAccessInst else { return true } + + return self.accessPath.isDistinct(from: otherBeginAccessInst.accessPath) + } + + guard areBeginAccessesSafe else { return false } + + var scope = InstructionRange(begin: self, ends: endAccessInstructions, context) + defer { scope.deinitialize() } + + for fullApplyInst in analyzedInstructions.fullApplies { + guard mayWriteToMemory && fullApplyInst.mayReadOrWrite(address: address, context.aliasAnalysis) || + !mayWriteToMemory && fullApplyInst.mayWrite(toAddress: address, context.aliasAnalysis) else { + continue + } + + // After hoisting the begin/end_access the apply will be within the scope, so it must not have a conflicting access. + if !scope.contains(fullApplyInst) { + return false + } + } + + switch accessPath.base { + case .class, .global: + for sideEffect in analyzedInstructions.loopSideEffects where sideEffect.mayRelease { + // Since a class might have a deinitializer, hoisting begin/end_access pair could violate + // exclusive access if the deinitializer accesses address used by begin_access. + if !scope.contains(sideEffect) { + return false + } + } + + return true + default: + return true + } + } +} + +private extension AccessPath { + /// Returns `true` if this access path is invariant in `loop`. + func isLoopInvariant(loop: Loop) -> Bool { + switch base { + case .box(let inst as Instruction), .class(let inst as Instruction), + .index(let inst as Instruction), + .pointer(let inst as Instruction), .stack(let inst as Instruction), + .storeBorrow(let inst as Instruction), + .tail(let inst as Instruction): + if loop.contains(block: inst.parentBlock) { + return false + } + case .global, .argument: + break + case .yield(let beginApplyResult): + if loop.contains(block: beginApplyResult.parentBlock) { + return false + } + case .unidentified: + return false + } + + return projectionPath.isConstant + } +} diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift index 36ceb2a8617f1..b141812d29bc4 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift @@ -364,13 +364,13 @@ private func constructObject(of allocRef: AllocRefInstBase, inInitializerOf global: GlobalVariable, _ storesToClassFields: [StoreInst], _ storesToTailElements: [StoreInst], _ context: FunctionPassContext) { - var cloner = StaticInitCloner(cloneTo: global, context) + var cloner = Cloner(cloneToGlobal: global, context) defer { cloner.deinitialize() } // Create the initializers for the fields var objectArgs = [Value]() for store in storesToClassFields { - objectArgs.append(cloner.clone(store.source as! SingleValueInstruction)) + objectArgs.append(cloner.cloneRecursivelyToGlobal(value: store.source as! SingleValueInstruction)) } let globalBuilder = Builder(staticInitializerOf: global, context) @@ -382,7 +382,7 @@ private func constructObject(of allocRef: AllocRefInstBase, for elementIdx in 0.. Bool { + return context.bridgedPassContext.canHoistArraySemanticsCall(self.bridged, toInst.bridged) + } + + func hoistArraySemanticsCall(before toInst: Instruction, _ context: some MutatingContext) { + context.bridgedPassContext.hoistArraySemanticsCall(self.bridged, toInst.bridged) // Internally updates dom tree. + context.notifyInstructionsChanged() + } +} + +extension Cloner where Context == FunctionPassContext { + func getOrCreateEntryBlock() -> BasicBlock { + if let entryBlock = targetFunction.blocks.first { + return entryBlock + } + return targetFunction.appendNewBlock(context) + } + + func cloneFunctionBody(from originalFunction: Function, entryBlockArguments: [Value]) { + entryBlockArguments.withBridgedValues { bridgedEntryBlockArgs in + let entryBlock = getOrCreateEntryBlock() + bridged.cloneFunctionBody(originalFunction.bridged, entryBlock.bridged, bridgedEntryBlockArgs) + } + } + + func cloneFunctionBody(from originalFunction: Function) { + bridged.cloneFunctionBody(originalFunction.bridged) + } +} + +func cloneFunction(from originalFunction: Function, toEmpty targetFunction: Function, _ context: FunctionPassContext) { + var cloner = Cloner(cloneToEmptyFunction: targetFunction, context) + defer { cloner.deinitialize() } + cloner.cloneFunctionBody(from: originalFunction) +} diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift index 18620eea7e914..d74055f4a2051 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift @@ -45,6 +45,11 @@ struct FunctionPassContext : MutatingContext { let bridgedPDT = bridgedPassContext.getPostDomTree() return PostDominatorTree(bridged: bridgedPDT) } + + var loopTree: LoopTree { + let bridgedLT = bridgedPassContext.getLoopTree() + return LoopTree(bridged: bridgedLT, context: self) + } func loadFunction(name: StaticString, loadCalleesRecursively: Bool) -> Function? { return name.withUTF8Buffer { (nameBuffer: UnsafeBufferPointer) in @@ -92,15 +97,11 @@ struct FunctionPassContext : MutatingContext { return String(taking: bridgedPassContext.mangleOutlinedVariable(function.bridged)) } - func mangle(withClosureArguments closureArgs: [Value], closureArgIndices: [Int], from applySiteCallee: Function) -> String { - closureArgs.withBridgedValues { bridgedClosureArgsRef in - closureArgIndices.withBridgedArrayRef{bridgedClosureArgIndicesRef in - String(taking: bridgedPassContext.mangleWithClosureArgs( - bridgedClosureArgsRef, - bridgedClosureArgIndicesRef, - applySiteCallee.bridged - )) - } + func mangle(withClosureArguments closureArgs: [(argumentIndex: Int, argumentValue: Value)], + from applySiteCallee: Function + ) -> String { + closureArgs.withBridgedArrayRef{ bridgedClosureArgs in + String(taking: bridgedPassContext.mangleWithClosureArgs(bridgedClosureArgs, applySiteCallee.bridged)) } } diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift index c89f350c7b80d..2744559fe12d7 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift @@ -106,6 +106,7 @@ private func registerSwiftPasses() { registerPass(tempLValueElimination, { tempLValueElimination.run($0) }) registerPass(generalClosureSpecialization, { generalClosureSpecialization.run($0) }) registerPass(autodiffClosureSpecialization, { autodiffClosureSpecialization.run($0) }) + registerPass(loopInvariantCodeMotionPass, { loopInvariantCodeMotionPass.run($0) }) // Instruction passes registerForSILCombine(BeginBorrowInst.self, { run(BeginBorrowInst.self, $0) }) diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt index 50dab7dcccb85..35ca4d97aae75 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt @@ -8,7 +8,6 @@ swift_compiler_sources(Optimizer AddressUtils.swift - SpecializationCloner.swift Devirtualization.swift EscapeUtils.swift FunctionSignatureTransforms.swift @@ -18,5 +17,4 @@ swift_compiler_sources(Optimizer LocalVariableUtils.swift OptUtils.swift OwnershipLiveness.swift - StaticInitCloner.swift ) diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift index d3a9302424cfe..da4d207669eb4 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift @@ -484,6 +484,22 @@ extension Instruction { } return false } + + /// Returns true if `otherInst` is in the same block and is strictly dominated by this instruction or + /// the parent block of the instruction dominates parent block of `otherInst`. + func dominates( + _ otherInst: Instruction, + _ domTree: DominatorTree + ) -> Bool { + if parentBlock == otherInst.parentBlock { + return dominatesInSameBlock(otherInst) + } else { + return parentBlock.dominates( + otherInst.parentBlock, + domTree + ) + } + } /// If this instruction uses a (single) existential archetype, i.e. it has a type-dependent operand, /// returns the concrete type if it is known. @@ -583,36 +599,22 @@ extension StoreInst { extension LoadInst { @discardableResult func trySplit(_ context: FunctionPassContext) -> Bool { - var elements = [Value]() - let builder = Builder(before: self, context) if type.isStruct { - if (type.nominal as! StructDecl).hasUnreferenceableStorage { - return false - } - guard let fields = type.getNominalFields(in: parentFunction) else { + guard !(type.nominal as! StructDecl).hasUnreferenceableStorage, + let fields = type.getNominalFields(in: parentFunction) else { return false } - for idx in 0.. LoadOwnership { @@ -623,6 +625,70 @@ extension LoadInst { return fieldValue.type.isTrivial(in: parentFunction) ? .trivial : self.loadOwnership } } + + func trySplit( + alongPath projectionPath: SmallProjectionPath, + _ context: FunctionPassContext + ) -> [LoadInst]? { + if projectionPath.isEmpty { + return nil + } + + let (fieldKind, index, pathRemainder) = projectionPath.pop() + + var elements: [LoadInst] + + switch fieldKind { + case .structField where type.isStruct: + guard !(type.nominal as! StructDecl).hasUnreferenceableStorage, + let fields = type.getNominalFields(in: parentFunction) else { + return nil + } + + elements = splitStruct(fields: fields, context) + case .tupleField where type.isTuple: + elements = splitTuple(context) + default: + return nil + } + + if let recursiveSplitLoad = elements[index].trySplit(alongPath: pathRemainder, context) { + elements.remove(at: index) + elements += recursiveSplitLoad + } + + return elements + } + + private func splitStruct(fields: NominalFieldsArray, _ context: FunctionPassContext) -> [LoadInst] { + var elements = [LoadInst]() + let builder = Builder(before: self, context) + + for idx in 0.. [LoadInst] { + var elements = [LoadInst]() + let builder = Builder(before: self, context) + + for idx in 0.. BasicBlock { - bridged.getClonedBasicBlock(originalBlock.bridged).block - } - - func cloneFunctionBody(from originalFunction: Function, entryBlockArguments: [Value]) { - entryBlockArguments.withBridgedValues { bridgedEntryBlockArgs in - bridged.cloneFunctionBody(originalFunction.bridged, self.entryBlock.bridged, bridgedEntryBlockArgs) - } - } - - func cloneFunctionBody(from originalFunction: Function) { - bridged.cloneFunctionBody(originalFunction.bridged) - } -} diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/StaticInitCloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/StaticInitCloner.swift deleted file mode 100644 index dcc9853052c6a..0000000000000 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/StaticInitCloner.swift +++ /dev/null @@ -1,78 +0,0 @@ -//===--- StaticInitCloner.swift --------------------------------------------==// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import SIL -import OptimizerBridging - -/// Clones the initializer value of a GlobalVariable. -/// -/// Used to transitively clone "constant" instructions, including their operands, -/// from or to the static initializer value of a GlobalVariable. -/// -struct StaticInitCloner { - private var bridged: BridgedCloner - private let context: Context - private let cloningIntoFunction: Bool - - init(cloneTo global: GlobalVariable, _ context: Context) { - self.bridged = BridgedCloner(global.bridged, context._bridged) - self.context = context - self.cloningIntoFunction = false - } - - init(cloneBefore inst: Instruction, _ context: Context) { - self.bridged = BridgedCloner(inst.bridged, context._bridged) - self.context = context - self.cloningIntoFunction = true - } - - mutating func deinitialize() { - bridged.destroy(context._bridged) - } - - /// Transitively clones `value` including its defining instruction's operands. - mutating func clone(_ value: Value) -> Value { - - if isCloned(value: value) { - return getClonedValue(of: value) - } - - if let beginAccess = value as? BeginAccessInst { - // Skip access instructions, which might be generated for UnsafePointer globals which point to other globals. - let clonedOperand = clone(beginAccess.address) - bridged.recordFoldedValue(beginAccess.bridged, clonedOperand.bridged) - return clonedOperand - } - - let inst = value.definingInstruction! - assert(!(inst is ScopedInstruction), "global init value must not contain a scoped instruction") - - for op in inst.operands { - _ = clone(op.value) - } - - bridged.clone(inst.bridged) - let clonedValue = getClonedValue(of: value) - if cloningIntoFunction { - context.notifyInstructionChanged(clonedValue.definingInstruction!) - } - return clonedValue - } - - mutating func getClonedValue(of value: Value) -> Value { - bridged.getClonedValue(value.bridged).value - } - - func isCloned(value: Value) -> Bool { - bridged.isValueCloned(value.bridged) - } -} diff --git a/SwiftCompilerSources/Sources/SIL/BasicBlock.swift b/SwiftCompilerSources/Sources/SIL/BasicBlock.swift index dcf5bb68474ec..c543559c24489 100644 --- a/SwiftCompilerSources/Sources/SIL/BasicBlock.swift +++ b/SwiftCompilerSources/Sources/SIL/BasicBlock.swift @@ -153,6 +153,14 @@ final public class BasicBlock : CustomStringConvertible, HasShortDescription, Ha return false } } + + public func isCriticalEdge(edgeIndex: Int) -> Bool { + if terminator.successors.count <= 1 { + return false + } else { + return !terminator.successors[edgeIndex].hasSinglePredecessor + } + } } /// The list of instructions in a BasicBlock. diff --git a/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift b/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift index 026e42fac92ab..30444bdb7723b 100644 --- a/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift +++ b/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift @@ -140,7 +140,7 @@ public struct Stack : CollectionLikeSequence { public mutating func deinitialize() { removeAll() } } -extension Stack { +public extension Stack { /// Mark a stack location for future iteration. /// /// TODO: Marker should be ~Escapable. @@ -155,7 +155,7 @@ extension Stack { let low: Marker let high: Marker - init(in stack: Stack, low: Marker, high: Marker) { + public init(in stack: Stack, low: Marker, high: Marker) { if low.slab.data == nil { assert(low.index == 0, "invalid empty stack marker") // `low == nil` and `high == nil` is a valid empty segment, @@ -173,7 +173,7 @@ extension Stack { self.high = high } - func makeIterator() -> Stack.Iterator { + public func makeIterator() -> Stack.Iterator { return Iterator(slab: low.slab, index: low.index, lastSlab: high.slab, endIndex: high.index) } @@ -219,3 +219,87 @@ extension Stack { } } } + +public struct StackWithCount : CollectionLikeSequence { + public private(set) var count = 0 + private var underlyingStack: Stack + + public typealias Iterator = Stack.Iterator + + public init(_ context: some Context) { + self.underlyingStack = Stack(context) + } + + public func makeIterator() -> Stack.Iterator { + underlyingStack.makeIterator() + } + + public var first: Element? { underlyingStack.first } + public var last: Element? { underlyingStack.last } + + public mutating func push(_ element: Element) { + count += 1 + underlyingStack.push(element) + } + + /// The same as `push` to provide an Array-like append API. + public mutating func append(_ element: Element) { push(element) } + + public mutating func append(contentsOf other: S) where S.Element == Element { + for elem in other { + append(elem) + } + } + + public var isEmpty: Bool { underlyingStack.isEmpty } + + public mutating func pop() -> Element? { + if underlyingStack.isEmpty { + return nil + } + + count -= 1 + return underlyingStack.pop() + } + + public mutating func removeAll() { + underlyingStack.removeAll() + } + + /// TODO: once we have move-only types, make this a real deinit. + public mutating func deinitialize() { removeAll() } +} + +public extension StackWithCount { + typealias Marker = Stack.Marker + + struct Segment : CollectionLikeSequence { + var underlyingSegment: Stack.Segment + + public init(in stack: StackWithCount, low: Marker, high: Marker) { + underlyingSegment = Stack.Segment(in: stack.underlyingStack, low: low, high: high) + } + + public func makeIterator() -> StackWithCount.Iterator { + return underlyingSegment.makeIterator() + } + } + + var top: Marker { underlyingStack.top } + + func assertValid(marker: Marker) { underlyingStack.assertValid(marker: marker) } + + mutating func withMarker( + _ body: (inout Stack, Marker) throws -> R) rethrows -> R { + return try underlyingStack.withMarker(body) + } + + mutating func withMarker( + pushElements body: (inout Stack) throws -> R, + withNewElements handleNewElements: ((Segment) -> ()) + ) rethrows -> R { + return try underlyingStack.withMarker(pushElements: body) { [self] segment in + handleNewElements(Segment(in: self, low: segment.low, high: segment.high)) + } + } +} diff --git a/SwiftCompilerSources/Sources/SIL/Effects.swift b/SwiftCompilerSources/Sources/SIL/Effects.swift index 5cd2ea66c19a3..311f4423c60aa 100644 --- a/SwiftCompilerSources/Sources/SIL/Effects.swift +++ b/SwiftCompilerSources/Sources/SIL/Effects.swift @@ -505,6 +505,14 @@ public struct SideEffects : CustomStringConvertible, NoReflectionChildren { /// This is true when the function (or a callee, transitively) contains a /// deinit barrier instruction. public var isDeinitBarrier: Bool + + public static var noEffects: GlobalEffects { + return GlobalEffects(memory: .noEffects, ownership: .noEffects, allocates: false, isDeinitBarrier: false) + } + + public var isOnlyReading: Bool { + return !memory.write && ownership == .noEffects && !allocates && !isDeinitBarrier + } /// When called with default arguments, it creates an "effect-free" GlobalEffects. public init(memory: Memory = Memory(read: false, write: false), @@ -643,6 +651,10 @@ public struct SideEffects : CustomStringConvertible, NoReflectionChildren { copy = copy || other.copy destroy = destroy || other.destroy } + + public static var noEffects: Ownership { + return Ownership(copy: false, destroy: false) + } public static var worstEffects: Ownership { Ownership(copy: true, destroy: true) diff --git a/SwiftCompilerSources/Sources/SIL/Instruction.swift b/SwiftCompilerSources/Sources/SIL/Instruction.swift index fd5e986ac4096..0c666bb2ec154 100644 --- a/SwiftCompilerSources/Sources/SIL/Instruction.swift +++ b/SwiftCompilerSources/Sources/SIL/Instruction.swift @@ -96,6 +96,11 @@ public class Instruction : CustomStringConvertible, Hashable { BridgedContext.moveInstructionBefore(bridged, otherInstruction.bridged) context.notifyInstructionsChanged() } + + public final func copy(before otherInstruction: Instruction, _ context: some MutatingContext) { + BridgedContext.copyInstructionBefore(bridged, otherInstruction.bridged) + context.notifyInstructionsChanged() + } public var mayTrap: Bool { false } @@ -177,6 +182,10 @@ public class Instruction : CustomStringConvertible, Hashable { public static func ==(lhs: Instruction, rhs: Instruction) -> Bool { lhs === rhs } + + public func isIdenticalTo(_ otherInst: Instruction) -> Bool { + return bridged.isIdenticalTo(otherInst.bridged) + } public func hash(into hasher: inout Hasher) { hasher.combine(ObjectIdentifier(self)) diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift b/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift index 546d5c7c30893..d0863f9b3d78d 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift @@ -392,6 +392,11 @@ public struct AccessPath : CustomStringConvertible, Hashable { public func isEqualOrContains(_ other: AccessPath) -> Bool { return getProjection(to: other) != nil } + + /// Returns true if this access contains `other` access and is not equal. + public func contains(_ other: AccessPath) -> Bool { + return !(getProjection(to: other)?.isEmpty ?? true) + } public var materializableProjectionPath: SmallProjectionPath? { if projectionPath.isMaterializable { diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt b/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt index f39a2a89ae0e3..59ba3af5f8912 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt @@ -9,6 +9,7 @@ swift_compiler_sources(SIL AccessUtils.swift BorrowUtils.swift + Cloner.swift ForwardingUtils.swift PhiUpdater.swift SequenceUtilities.swift diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift new file mode 100644 index 0000000000000..ffdf7b1e94dec --- /dev/null +++ b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift @@ -0,0 +1,141 @@ +//===--- Cloner.swift ------------------------------------------------------==// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import OptimizerBridging +import SILBridging + +/// Clones the initializer value of a GlobalVariable. +/// +/// Used to transitively clone "constant" instructions, including their operands, +/// from or to the static initializer value of a GlobalVariable. +/// +public struct Cloner { + public var bridged: BridgedCloner + public let context: Context + + public enum GetClonedResult { + case defaultValue + case customValue(Value) + case stopCloning + } + + public enum Target { + case function(Function) + case global(GlobalVariable) + } + public let target: Target + + public init(cloneToGlobal: GlobalVariable, _ context: Context) { + self.bridged = BridgedCloner(cloneToGlobal.bridged, context._bridged) + self.context = context + self.target = .global(cloneToGlobal) + } + + public init(cloneBefore inst: Instruction, _ context: Context) { + self.bridged = BridgedCloner(inst.bridged, context._bridged) + self.context = context + self.target = .function(inst.parentFunction) + } + + public init(cloneToEmptyFunction: Function, _ context: Context) { + self.bridged = BridgedCloner(cloneToEmptyFunction.bridged, context._bridged) + self.context = context + self.target = .function(cloneToEmptyFunction) + } + + public mutating func deinitialize() { + bridged.destroy(context._bridged) + } + + public var targetFunction: Function { + guard case .function(let function) = target else { + fatalError("expected cloning into a function") + } + return function + } + + public mutating func clone(instruction: Instruction) -> Instruction { + let cloned = bridged.clone(instruction.bridged).instruction + if case .function = target { + context.notifyInstructionChanged(cloned) + context.notifyInstructionsChanged() + } + return cloned + } + + public mutating func cloneRecursivelyToGlobal(value: Value) -> Value { + guard let cloned = cloneRecursively(value: value, customGetCloned: { value, cloner in + guard let beginAccess = value as? BeginAccessInst else { + return .defaultValue + } + + // Skip access instructions, which might be generated for UnsafePointer globals which point to other globals. + let clonedOperand = cloner.cloneRecursivelyToGlobal(value: beginAccess.address) + cloner.recordFoldedValue(beginAccess, mappedTo: clonedOperand) + return .customValue(clonedOperand) + }) else { + fatalError("Clone recursively to global shouldn't bail.") + } + + return cloned + } + + /// Transitively clones `value` including its defining instruction's operands. + public mutating func cloneRecursively(value: Value, customGetCloned: (Value, inout Cloner) -> GetClonedResult) -> Value? { + if isCloned(value: value) { + return getClonedValue(of: value) + } + + switch customGetCloned(value, &self) { + case .customValue(let base): + return base + case .stopCloning: + return nil + case .defaultValue: + break + } + + guard let inst = value.definingInstruction else { + fatalError("expected instruction to clone or already cloned value") + } + + for op in inst.operands { + if cloneRecursively(value: op.value, customGetCloned: customGetCloned) == nil { + return nil + } + } + + let cloned = clone(instruction: inst) + if let svi = cloned as? SingleValueInstruction { + return svi + } else if let originalMvi = value as? MultipleValueInstructionResult { + return cloned.results[originalMvi.index] + } + fatalError("unexpected instruction kind") + } + + public mutating func getClonedValue(of originalValue: Value) -> Value { + bridged.getClonedValue(originalValue.bridged).value + } + + public func isCloned(value: Value) -> Bool { + bridged.isValueCloned(value.bridged) + } + + public func getClonedBlock(for originalBlock: BasicBlock) -> BasicBlock { + bridged.getClonedBasicBlock(originalBlock.bridged).block + } + + public func recordFoldedValue(_ origValue: Value, mappedTo mappedValue: Value) { + bridged.recordFoldedValue(origValue.bridged, mappedValue.bridged) + } +} diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift b/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift index ea94f2771e83f..cfb206bd27f66 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift @@ -534,6 +534,18 @@ public struct SmallProjectionPath : Hashable, CustomStringConvertible, NoReflect return false } } + + public var isConstant: Bool { + let (kind, _, subPath) = pop() + switch kind { + case .root: + return true + case .structField, .tupleField, .enumCase, .classField, .existential, .indexedElement: + return subPath.isConstant + default: + return false + } + } } //===----------------------------------------------------------------------===// diff --git a/include/swift/SIL/SILBridging.h b/include/swift/SIL/SILBridging.h index bf38cec201f12..9fead48f327ba 100644 --- a/include/swift/SIL/SILBridging.h +++ b/include/swift/SIL/SILBridging.h @@ -60,6 +60,9 @@ class SymbolicValueBumpAllocator; class ConstExprEvaluator; class SILWitnessTable; class SILDefaultWitnessTable; +class SILLoopInfo; +class SILLoop; +class BridgedClonerImpl; class SILDebugLocation; class NominalTypeDecl; class VarDecl; @@ -75,11 +78,25 @@ class SILLocation; class BasicBlockSet; class NodeSet; class OperandSet; -class ClonerWithFixedLocation; class FixedSizeSlabPayload; class FixedSizeSlab; } +struct BridgedLoop { + swift::SILLoop * _Nonnull l; + + BRIDGED_INLINE SwiftInt getInnerLoopCount() const; + BRIDGED_INLINE BridgedLoop getInnerLoop(SwiftInt index) const; + + BRIDGED_INLINE SwiftInt getBasicBlockCount() const; + BRIDGED_INLINE BridgedBasicBlock getBasicBlock(SwiftInt index) const; + + BRIDGED_INLINE OptionalBridgedBasicBlock getPreheader() const; + BRIDGED_INLINE BridgedBasicBlock getHeader() const; + + BRIDGED_INLINE bool contains(BridgedBasicBlock block) const; +}; + bool swiftModulesInitialized(); void registerBridgedClass(BridgedStringRef className, SwiftMetatype metatype); @@ -678,6 +695,7 @@ struct BridgedInstruction { bool maySynchronize() const; bool mayBeDeinitBarrierNotConsideringSideEffects() const; BRIDGED_INLINE bool shouldBeForwarding() const; + BRIDGED_INLINE bool isIdenticalTo(BridgedInstruction inst) const; // =========================================================================// // Generalized instruction subclasses @@ -1279,7 +1297,7 @@ struct BridgedBuilder{ BridgedArgumentConvention calleeConvention, BridgedSubstitutionMap bridgedSubstitutionMap = BridgedSubstitutionMap(), bool hasUnknownIsolation = true, - bool isOnStack = false) const; + bool isOnStack = false) const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createBranch(BridgedBasicBlock destBlock, BridgedValueArray arguments) const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createUnreachable() const; @@ -1319,7 +1337,7 @@ struct BridgedBuilder{ BridgedASTType::MetatypeRepresentation representation) const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createEndCOWMutation(BridgedValue instance, bool keepUnique) const; - SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createEndCOWMutationAddr(BridgedValue instance) const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createEndCOWMutationAddr(BridgedValue instance) const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createMarkDependence( BridgedValue value, BridgedValue base, BridgedInstruction::MarkDependenceKind dependenceKind) const; @@ -1376,18 +1394,6 @@ struct BridgedOperandSet { SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedFunction getFunction() const; }; -struct BridgedCloner { - swift::ClonerWithFixedLocation * _Nonnull cloner; - - BridgedCloner(BridgedGlobalVar var, BridgedContext context); - BridgedCloner(BridgedInstruction inst, BridgedContext context); - void destroy(BridgedContext context); - SWIFT_IMPORT_UNSAFE BridgedValue getClonedValue(BridgedValue v); - bool isValueCloned(BridgedValue v) const; - void clone(BridgedInstruction inst); - void recordFoldedValue(BridgedValue origValue, BridgedValue mappedValue); -}; - struct BridgedContext { swift::SILContext * _Nonnull context; @@ -1465,6 +1471,7 @@ struct BridgedContext { BRIDGED_INLINE void eraseInstruction(BridgedInstruction inst, bool salvageDebugInfo) const; BRIDGED_INLINE void eraseBlock(BridgedBasicBlock block) const; static BRIDGED_INLINE void moveInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst); + static BRIDGED_INLINE void copyInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst); // SSAUpdater @@ -1501,6 +1508,25 @@ struct BridgedContext { SWIFT_IMPORT_UNSAFE BRIDGED_INLINE Slab freeSlab(Slab slab) const; }; +struct BridgedCloner { + swift::BridgedClonerImpl * _Nonnull cloner; + + BridgedCloner(BridgedGlobalVar var, BridgedContext context); + BridgedCloner(BridgedInstruction inst, BridgedContext context); + BridgedCloner(BridgedFunction emptyFunction, BridgedContext context); + void destroy(BridgedContext context); + SWIFT_IMPORT_UNSAFE BridgedFunction getCloned() const; + SWIFT_IMPORT_UNSAFE BridgedBasicBlock getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const; + void cloneFunctionBody(BridgedFunction originalFunction, BridgedBasicBlock clonedEntryBlock, + BridgedValueArray clonedEntryBlockArgs) const; + void cloneFunctionBody(BridgedFunction originalFunction) const; + SWIFT_IMPORT_UNSAFE BridgedValue getClonedValue(BridgedValue v); + bool isValueCloned(BridgedValue v) const; + void recordClonedInstruction(BridgedInstruction origInst, BridgedInstruction clonedInst) const; + void recordFoldedValue(BridgedValue orig, BridgedValue mapped) const; + BridgedInstruction clone(BridgedInstruction inst); +}; + struct BridgedVerifier { typedef void (* _Nonnull VerifyFunctionFn)(BridgedContext, BridgedFunction); diff --git a/include/swift/SIL/SILBridgingImpl.h b/include/swift/SIL/SILBridgingImpl.h index 0d2cbef48701e..4d21690294fca 100644 --- a/include/swift/SIL/SILBridgingImpl.h +++ b/include/swift/SIL/SILBridgingImpl.h @@ -686,7 +686,7 @@ BridgedStringRef BridgedFunction::getName() const { } BridgedLocation BridgedFunction::getLocation() const { - return {swift::SILDebugLocation(getFunction()->getLocation(), getFunction()->getDebugScope())}; + return {swift::SILDebugLocation(getFunction()->getLocation(), getFunction()->getDebugScope())}; } bool BridgedFunction::isAccessor() const { @@ -1070,6 +1070,10 @@ bool BridgedInstruction::shouldBeForwarding() const { llvm::isa(unbridged()); } +bool BridgedInstruction::isIdenticalTo(BridgedInstruction inst) const { + return unbridged()->isIdenticalTo(inst.unbridged()); +} + SwiftInt BridgedInstruction::MultipleValueInstruction_getNumResults() const { return getAs()->getNumResults(); } @@ -1333,7 +1337,7 @@ BridgedGenericSpecializationInformation BridgedInstruction::ApplyInst_getSpecial } bool BridgedInstruction::TryApplyInst_getNonAsync() const { - return getAs()->isNonAsync(); + return getAs()->isNonAsync(); } BridgedGenericSpecializationInformation BridgedInstruction::TryApplyInst_getSpecializationInfo() const { @@ -1349,7 +1353,7 @@ BridgedDeclRef BridgedInstruction::WitnessMethodInst_getMember() const { } BridgedCanType BridgedInstruction::WitnessMethodInst_getLookupType() const { - return getAs()->getLookupType(); + return getAs()->getLookupType(); } BridgedDeclObj BridgedInstruction::WitnessMethodInst_getLookupProtocol() const { @@ -1602,11 +1606,11 @@ void BridgedInstruction::CheckedCastBranch_updateSourceFormalTypeFromOperandLowe } BridgedCanType BridgedInstruction::UnconditionalCheckedCast_getSourceFormalType() const { - return {getAs()->getSourceFormalType()}; + return {getAs()->getSourceFormalType()}; } BridgedCanType BridgedInstruction::UnconditionalCheckedCast_getTargetFormalType() const { - return {getAs()->getTargetFormalType()}; + return {getAs()->getTargetFormalType()}; } BridgedInstruction::CheckedCastInstOptions @@ -1617,11 +1621,11 @@ BridgedInstruction::UnconditionalCheckedCast_getCheckedCastOptions() const { } BridgedCanType BridgedInstruction::UnconditionalCheckedCastAddr_getSourceFormalType() const { - return {getAs()->getSourceFormalType()}; + return {getAs()->getSourceFormalType()}; } BridgedCanType BridgedInstruction::UnconditionalCheckedCastAddr_getTargetFormalType() const { - return {getAs()->getTargetFormalType()}; + return {getAs()->getTargetFormalType()}; } BridgedInstruction::CheckedCastInstOptions @@ -1651,7 +1655,7 @@ BridgedCanType BridgedInstruction::CheckedCastAddrBranch_getSourceFormalType() c } BridgedCanType BridgedInstruction::CheckedCastAddrBranch_getTargetFormalType() const { - return {getAs()->getTargetFormalType()}; + return {getAs()->getTargetFormalType()}; } BridgedBasicBlock BridgedInstruction::CheckedCastAddrBranch_getSuccessBlock() const { @@ -2508,7 +2512,7 @@ BridgedInstruction BridgedBuilder::createThinToThickFunction(BridgedValue fn, Br resultType.unbridged())}; } -BridgedInstruction BridgedBuilder::createPartialApply(BridgedValue funcRef, +BridgedInstruction BridgedBuilder::createPartialApply(BridgedValue funcRef, BridgedValueArray bridgedCapturedArgs, BridgedArgumentConvention calleeConvention, BridgedSubstitutionMap bridgedSubstitutionMap, @@ -2523,7 +2527,7 @@ BridgedInstruction BridgedBuilder::createPartialApply(BridgedValue funcRef, : swift::SILFunctionTypeIsolation::forErased(), isOnStack ? swift::PartialApplyInst::OnStack : swift::PartialApplyInst::NotOnStack)}; -} +} BridgedInstruction BridgedBuilder::createBranch(BridgedBasicBlock destBlock, BridgedValueArray arguments) const { llvm::SmallVector argValues; @@ -2940,6 +2944,10 @@ void BridgedContext::moveInstructionBefore(BridgedInstruction inst, BridgedInstr swift::SILBasicBlock::moveInstruction(inst.unbridged(), beforeInst.unbridged()); } +void BridgedContext::copyInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst) { + inst.unbridged()->clone(beforeInst.unbridged()); +} + OptionalBridgedFunction BridgedContext::lookupStdlibFunction(BridgedStringRef name) const { return {context->lookupStdlibFunction(name.unbridged())}; } diff --git a/include/swift/SILOptimizer/Analysis/ArrayCallKind.h b/include/swift/SILOptimizer/Analysis/ArrayCallKind.h new file mode 100644 index 0000000000000..f3db620d0d162 --- /dev/null +++ b/include/swift/SILOptimizer/Analysis/ArrayCallKind.h @@ -0,0 +1,45 @@ +//===--- ArrayCallKind.h -------------------------------------- -*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef ARRAY_CALL_KIND_H +#define ARRAY_CALL_KIND_H + +/// The kind of array operation identified by looking at the semantics attribute +/// of the called function. +enum class ArrayCallKind { + kNone = 0, + kArrayPropsIsNativeTypeChecked, + kCheckSubscript, + kCheckIndex, + kGetCount, + kGetCapacity, + kGetElement, + kGetElementAddress, + kMakeMutable, + kEndMutation, + kMutateUnknown, + kReserveCapacityForAppend, + kWithUnsafeMutableBufferPointer, + kAppendContentsOf, + kAppendElement, + // The following two semantic function kinds return the result @owned + // instead of operating on self passed as parameter. If you are adding + // a function, and it has a self parameter, make sure that it is defined + // before this comment. + kArrayInit, + kArrayInitEmpty, + kArrayUninitialized, + kArrayUninitializedIntrinsic, + kArrayFinalizeIntrinsic +}; + +#endif diff --git a/include/swift/SILOptimizer/Analysis/ArraySemantic.h b/include/swift/SILOptimizer/Analysis/ArraySemantic.h index 7bbca9d7c7e50..0b144562dea94 100644 --- a/include/swift/SILOptimizer/Analysis/ArraySemantic.h +++ b/include/swift/SILOptimizer/Analysis/ArraySemantic.h @@ -14,40 +14,12 @@ #define SWIFT_SILOPTIMIZER_ANALYSIS_ARRAYSEMANTIC_H #include "swift/SIL/SILInstruction.h" +#include "swift/SILOptimizer/Analysis/ArrayCallKind.h" namespace swift { class DominanceInfo; -/// The kind of array operation identified by looking at the semantics attribute -/// of the called function. -enum class ArrayCallKind { - kNone = 0, - kArrayPropsIsNativeTypeChecked, - kCheckSubscript, - kCheckIndex, - kGetCount, - kGetCapacity, - kGetElement, - kGetElementAddress, - kMakeMutable, - kEndMutation, - kMutateUnknown, - kReserveCapacityForAppend, - kWithUnsafeMutableBufferPointer, - kAppendContentsOf, - kAppendElement, - // The following two semantic function kinds return the result @owned - // instead of operating on self passed as parameter. If you are adding - // a function, and it has a self parameter, make sure that it is defined - // before this comment. - kArrayInit, - kArrayInitEmpty, - kArrayUninitialized, - kArrayUninitializedIntrinsic, - kArrayFinalizeIntrinsic -}; - /// Return true is the given function is an array semantics call. ArrayCallKind getArraySemanticsKind(SILFunction *f); diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index efab99a89aef6..e553c96cdb755 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -19,6 +19,7 @@ /// See include guidelines and caveats in `BasicBridging.h`. #include "swift/AST/ASTBridging.h" #include "swift/SIL/SILBridging.h" +#include "swift/SILOptimizer/Analysis/ArrayCallKind.h" #ifndef NOT_COMPILED_WITH_SWIFT_PURE_BRIDGING_MODE @@ -37,11 +38,14 @@ SWIFT_BEGIN_NULLABILITY_ANNOTATIONS namespace swift { class AliasAnalysis; +class ArraySemanticsCall; class BasicCalleeAnalysis; class CalleeList; class DeadEndBlocks; class DominanceInfo; class PostDominanceInfo; +class SILLoopInfo; +class SILLoop; class SwiftPassInvocation; class SILVTable; class SpecializationCloner; @@ -111,6 +115,8 @@ struct BridgedDomTree { swift::DominanceInfo * _Nonnull di; BRIDGED_INLINE bool dominates(BridgedBasicBlock dominating, BridgedBasicBlock dominated) const; + BRIDGED_INLINE SwiftInt getNumberOfChildren(BridgedBasicBlock bb) const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedBasicBlock getChildAt(BridgedBasicBlock bb, SwiftInt index) const; }; struct BridgedPostDomTree { @@ -119,15 +125,13 @@ struct BridgedPostDomTree { BRIDGED_INLINE bool postDominates(BridgedBasicBlock dominating, BridgedBasicBlock dominated) const; }; -struct BridgedSpecializationCloner { - swift::SpecializationCloner * _Nonnull cloner; - - SWIFT_IMPORT_UNSAFE BridgedSpecializationCloner(BridgedFunction emptySpecializedFunction); - SWIFT_IMPORT_UNSAFE BridgedFunction getCloned() const; - SWIFT_IMPORT_UNSAFE BridgedBasicBlock getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const; - void cloneFunctionBody(BridgedFunction originalFunction, BridgedBasicBlock clonedEntryBlock, - BridgedValueArray clonedEntryBlockArgs) const; - void cloneFunctionBody(BridgedFunction originalFunction) const; +struct BridgedLoopTree { + swift::SILLoopInfo * _Nonnull li; + + BRIDGED_INLINE SwiftInt getTopLevelLoopCount() const; + BRIDGED_INLINE BridgedLoop getLoop(SwiftInt index) const; + + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedBasicBlock splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const; }; struct BridgedPassContext { @@ -146,6 +150,16 @@ struct BridgedPassContext { SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedDeadEndBlocksAnalysis getDeadEndBlocksAnalysis() const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedDomTree getDomTree() const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedPostDomTree getPostDomTree() const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedDeclObj getSwiftArrayDecl() const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedDeclObj getSwiftMutableSpanDecl() const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedLoopTree getLoopTree() const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedLoop getLoop() const; + + // Array semantics call + + static BRIDGED_INLINE ArrayCallKind getArraySemanticsCallKind(BridgedInstruction inst); + BRIDGED_INLINE bool canHoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction toInst) const; + BRIDGED_INLINE void hoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction beforeInst) const; // AST @@ -173,8 +187,7 @@ struct BridgedPassContext { BridgedOwnedString mangleOutlinedVariable(BridgedFunction function) const; BridgedOwnedString mangleAsyncRemoved(BridgedFunction function) const; BridgedOwnedString mangleWithDeadArgs(BridgedArrayRef bridgedDeadArgIndices, BridgedFunction function) const; - BridgedOwnedString mangleWithClosureArgs(BridgedValueArray closureArgs, - BridgedArrayRef closureArgIndices, + BridgedOwnedString mangleWithClosureArgs(BridgedArrayRef closureArgIndices, BridgedFunction applySiteCallee) const; BridgedOwnedString mangleWithBoxToStackPromotedArgs(BridgedArrayRef bridgedPromotedArgIndices, BridgedFunction bridgedOriginalFunction) const; diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index 91f560c440360..f700122cd64cd 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -21,9 +21,11 @@ #include "swift/Demangling/Demangle.h" #include "swift/SILOptimizer/Analysis/AliasAnalysis.h" +#include "swift/SILOptimizer/Analysis/ArraySemantic.h" #include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h" #include "swift/SILOptimizer/Analysis/DeadEndBlocksAnalysis.h" #include "swift/SILOptimizer/Analysis/DominanceAnalysis.h" +#include "swift/SILOptimizer/Analysis/LoopAnalysis.h" #include "swift/SILOptimizer/OptimizerBridging.h" #include "swift/SILOptimizer/PassManager/PassManager.h" #include "swift/SILOptimizer/PassManager/Transforms.h" @@ -83,10 +85,62 @@ bool BridgedDomTree::dominates(BridgedBasicBlock dominating, BridgedBasicBlock d return di->dominates(dominating.unbridged(), dominated.unbridged()); } +SwiftInt BridgedDomTree::getNumberOfChildren(BridgedBasicBlock bb) const { + return di->getNode(bb.unbridged())->getNumChildren(); +} + +BridgedBasicBlock BridgedDomTree::getChildAt(BridgedBasicBlock bb, SwiftInt index) const { + return {di->getNode(bb.unbridged())->begin()[index]->getBlock()}; +} + bool BridgedPostDomTree::postDominates(BridgedBasicBlock dominating, BridgedBasicBlock dominated) const { return pdi->dominates(dominating.unbridged(), dominated.unbridged()); } +//===----------------------------------------------------------------------===// +// BridgedLoopTree, BridgedLoop +//===----------------------------------------------------------------------===// + +SwiftInt BridgedLoopTree::getTopLevelLoopCount() const { + return li->end() - li->begin(); +} + +BridgedLoop BridgedLoopTree::getLoop(SwiftInt index) const { + return {li->begin()[index]}; +} + +BridgedBasicBlock BridgedLoopTree::splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const { + return {swift::splitEdge(bb.unbridged()->getTerminator(), edgeIndex, domTree.di, li)}; +} + +SwiftInt BridgedLoop::getInnerLoopCount() const { + return l->end() - l->begin(); +} + +BridgedLoop BridgedLoop::getInnerLoop(SwiftInt index) const { + return {l->begin()[index]}; +} + +SwiftInt BridgedLoop::getBasicBlockCount() const { + return l->getBlocks().size(); +} + +BridgedBasicBlock BridgedLoop::getBasicBlock(SwiftInt index) const { + return {l->getBlocks()[index]}; +} + +OptionalBridgedBasicBlock BridgedLoop::getPreheader() const { + return {l->getLoopPreheader()}; +} + +BridgedBasicBlock BridgedLoop::getHeader() const { + return {l->getHeader()}; +} + +bool BridgedLoop::contains(BridgedBasicBlock block) const { + return l->contains(block.unbridged()); +} + //===----------------------------------------------------------------------===// // BridgedPassContext //===----------------------------------------------------------------------===// @@ -136,6 +190,38 @@ BridgedPostDomTree BridgedPassContext::getPostDomTree() const { return {pda->get(invocation->getFunction())}; } +BridgedLoopTree BridgedPassContext::getLoopTree() const { + auto *lt = invocation->getPassManager()->getAnalysis(); + return {lt->get(invocation->getFunction())}; +} + +BridgedDeclObj BridgedPassContext::getSwiftArrayDecl() const { + swift::SILModule *mod = invocation->getPassManager()->getModule(); + return {mod->getASTContext().getArrayDecl()}; +} + +BridgedDeclObj BridgedPassContext::getSwiftMutableSpanDecl() const { + swift::SILModule *mod = invocation->getPassManager()->getModule(); + return {mod->getASTContext().getMutableSpanDecl()}; +} + +// Array semantics call + +ArrayCallKind BridgedPassContext::getArraySemanticsCallKind(BridgedInstruction inst) { + swift::ArraySemanticsCall semCall(inst.unbridged()); + return semCall.getKind(); +} + +bool BridgedPassContext::canHoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction toInst) const { + swift::ArraySemanticsCall semCall(inst.unbridged()); + return semCall.canHoist(toInst.unbridged(), getDomTree().di); +} + +void BridgedPassContext::hoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction beforeInst) const { + swift::ArraySemanticsCall semCall(inst.unbridged()); + semCall.hoist(beforeInst.unbridged(), getDomTree().di); +} + // AST SWIFT_IMPORT_UNSAFE BRIDGED_INLINE diff --git a/include/swift/SILOptimizer/PassManager/Passes.def b/include/swift/SILOptimizer/PassManager/Passes.def index a87327b70711a..7c9cc72eb7d15 100644 --- a/include/swift/SILOptimizer/PassManager/Passes.def +++ b/include/swift/SILOptimizer/PassManager/Passes.def @@ -51,7 +51,7 @@ /// This macro follows the same conventions as PASS(Id, Tag, Description), /// but is used for IRGen passes which are built outside of the /// SILOptimizer library. -/// +/// /// An IRGen pass is created by IRGen and needs to be registered with the pass /// manager dynamically. #ifndef IRGEN_PASS @@ -148,6 +148,8 @@ PASS(TempRValueElimination, "temp-rvalue-elimination", "Remove short-lived immutable temporary copies") PASS(TempLValueElimination, "temp-lvalue-elimination", "Remove short-lived immutable temporary l-values") +PASS(LoopInvariantCodeMotion, "loop-invariant-code-motion", + "New Loop Invariant Code Motion") // NOTE - ExperimentalSwiftBasedClosureSpecialization and AutodiffClosureSpecialization are a WIP PASS(ExperimentalSwiftBasedClosureSpecialization, "experimental-swift-based-closure-specialization", @@ -331,8 +333,6 @@ LEGACY_PASS(MandatoryARCOpts, "mandatory-arc-opts", "Mandatory ARC Optimization") LEGACY_PASS(HighLevelCSE, "high-level-cse", "Common Subexpression Elimination on High-Level SIL") -LEGACY_PASS(HighLevelLICM, "high-level-licm", - "Loop Invariant Code Motion in High-Level SIL") LEGACY_PASS(IVInfoPrinter, "iv-info-printer", "Print Induction Variable Information for Testing") LEGACY_PASS(LowerHopToActor, "lower-hop-to-actor", @@ -345,8 +345,6 @@ LEGACY_PASS(JumpThreadSimplifyCFG, "jumpthread-simplify-cfg", "Simplify CFG via Jump Threading") LEGACY_PASS(LetPropertiesOpt, "let-properties-opt", "Let Property Optimization") -LEGACY_PASS(LICM, "licm", - "Loop Invariant Code Motion") LEGACY_PASS(LateCodeMotion, "late-codemotion", "Late Code Motion with Release Hoisting") LEGACY_PASS(LateDeadFunctionAndGlobalElimination, "late-deadfuncelim", diff --git a/lib/SIL/Utils/SILBridging.cpp b/lib/SIL/Utils/SILBridging.cpp index 75eb2eec85244..81f59168328e8 100644 --- a/lib/SIL/Utils/SILBridging.cpp +++ b/lib/SIL/Utils/SILBridging.cpp @@ -546,59 +546,96 @@ BridgedInstruction BridgedBuilder::createSwitchEnumAddrInst(BridgedValue enumAdd } //===----------------------------------------------------------------------===// -// BridgedCloner +// BridgedCloner //===----------------------------------------------------------------------===// // Need to put ClonerWithFixedLocation into namespace swift to forward reference -// it in SILBridging.h. +// it in OptimizerBridging.h. namespace swift { -class ClonerWithFixedLocation : public SILCloner { - friend class SILInstructionVisitor; - friend class SILCloner; +class BridgedClonerImpl : public SILCloner { + friend class SILInstructionVisitor; + friend class SILCloner; - SILDebugLocation insertLoc; + bool hasFixedLocation; + union { + SILDebugLocation fixedLocation; + ScopeCloner scopeCloner; + }; -public: - ClonerWithFixedLocation(SILGlobalVariable *gVar) - : SILCloner(gVar), - insertLoc(ArtificialUnreachableLocation(), nullptr) {} + SILInstruction *result = nullptr; - ClonerWithFixedLocation(SILInstruction *insertionPoint) - : SILCloner(*insertionPoint->getFunction()), - insertLoc(insertionPoint->getDebugLocation()) { +public: + BridgedClonerImpl(SILGlobalVariable *gVar) + : SILCloner(gVar), + hasFixedLocation(true), + fixedLocation(ArtificialUnreachableLocation(), nullptr) {} + + BridgedClonerImpl(SILInstruction *insertionPoint) + : SILCloner(*insertionPoint->getFunction()), + hasFixedLocation(true), + fixedLocation(insertionPoint->getDebugLocation()) { Builder.setInsertionPoint(insertionPoint); } + BridgedClonerImpl(SILFunction &emptyFunction) + : SILCloner(emptyFunction), + hasFixedLocation(false), + scopeCloner(ScopeCloner(emptyFunction)) {} + + ~BridgedClonerImpl() { + if (hasFixedLocation) { + fixedLocation.~SILDebugLocation(); + } else { + scopeCloner.~ScopeCloner(); + } + } + SILValue getClonedValue(SILValue v) { return getMappedValue(v); } - void cloneInst(SILInstruction *inst) { + SILInstruction *cloneInst(SILInstruction *inst) { + result = nullptr; visit(inst); + ASSERT(result && "instruction not cloned"); + return result; } -protected: - SILLocation remapLocation(SILLocation loc) { - return insertLoc.getLocation(); + if (hasFixedLocation) + return fixedLocation.getLocation(); + return loc; } const SILDebugScope *remapScope(const SILDebugScope *DS) { - return insertLoc.getScope(); + if (hasFixedLocation) + return fixedLocation.getScope(); + return scopeCloner.getOrCreateClonedScope(DS); + } + + void postProcess(SILInstruction *Orig, SILInstruction *Cloned) { + result = Cloned; + SILCloner::postProcess(Orig, Cloned); } + }; } // namespace swift BridgedCloner::BridgedCloner(BridgedGlobalVar var, BridgedContext context) - : cloner(new ClonerWithFixedLocation(var.getGlobal())) { + : cloner(new BridgedClonerImpl(var.getGlobal())) { context.context->notifyNewCloner(); } BridgedCloner::BridgedCloner(BridgedInstruction inst, BridgedContext context) - : cloner(new ClonerWithFixedLocation(inst.unbridged())) { + : cloner(new BridgedClonerImpl(inst.unbridged())) { + context.context->notifyNewCloner(); +} + +BridgedCloner::BridgedCloner(BridgedFunction emptyFunction, BridgedContext context) + : cloner(new BridgedClonerImpl(*emptyFunction.getFunction())) { context.context->notifyNewCloner(); } @@ -608,6 +645,10 @@ void BridgedCloner::destroy(BridgedContext context) { context.context->notifyClonerDestroyed(); } +BridgedFunction BridgedCloner::getCloned() const { + return { &cloner->getBuilder().getFunction() }; +} + BridgedValue BridgedCloner::getClonedValue(BridgedValue v) { return {cloner->getClonedValue(v.getSILValue())}; } @@ -616,12 +657,32 @@ bool BridgedCloner::isValueCloned(BridgedValue v) const { return cloner->isValueCloned(v.getSILValue()); } -void BridgedCloner::clone(BridgedInstruction inst) { - cloner->cloneInst(inst.unbridged()); +void BridgedCloner::recordClonedInstruction(BridgedInstruction origInst, BridgedInstruction clonedInst) const { + cloner->recordClonedInstruction(origInst.unbridged(), clonedInst.unbridged()); +} + +void BridgedCloner::recordFoldedValue(BridgedValue orig, BridgedValue mapped) const { + cloner->recordFoldedValue(orig.getSILValue(), mapped.getSILValue()); +} + +BridgedInstruction BridgedCloner::clone(BridgedInstruction inst) { + return {cloner->cloneInst(inst.unbridged())->asSILNode()}; +} + +BridgedBasicBlock BridgedCloner::getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const { + return { cloner->getOpBasicBlock(originalBasicBlock.unbridged()) }; +} + +void BridgedCloner::cloneFunctionBody(BridgedFunction originalFunction, + BridgedBasicBlock clonedEntryBlock, + BridgedValueArray clonedEntryBlockArgs) const { + llvm::SmallVector clonedEntryBlockArgsStorage; + auto clonedEntryBlockArgsArrayRef = clonedEntryBlockArgs.getValues(clonedEntryBlockArgsStorage); + cloner->cloneFunctionBody(originalFunction.getFunction(), clonedEntryBlock.unbridged(), clonedEntryBlockArgsArrayRef); } -void BridgedCloner::recordFoldedValue(BridgedValue origValue, BridgedValue mappedValue) { - cloner->recordFoldedValue(origValue.getSILValue(), mappedValue.getSILValue()); +void BridgedCloner::cloneFunctionBody(BridgedFunction originalFunction) const { + cloner->cloneFunction(originalFunction.getFunction()); } //===----------------------------------------------------------------------===// diff --git a/lib/SILOptimizer/LoopTransforms/CMakeLists.txt b/lib/SILOptimizer/LoopTransforms/CMakeLists.txt index d0f4068d880c2..b0de47af2f670 100644 --- a/lib/SILOptimizer/LoopTransforms/CMakeLists.txt +++ b/lib/SILOptimizer/LoopTransforms/CMakeLists.txt @@ -4,5 +4,4 @@ target_sources(swiftSILOptimizer PRIVATE COWArrayOpt.cpp LoopRotate.cpp LoopUnroll.cpp - LICM.cpp ForEachLoopUnroll.cpp) diff --git a/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp b/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp index 0b12222f412b0..b9a1314a2fd2e 100644 --- a/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp +++ b/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp @@ -337,7 +337,7 @@ void ArrayInfo::classifyUsesOfArray(SILValue arrayValue) { if (arrayOp.doesNotChangeArray()) continue; - if (arrayOp.getKind() == swift::ArrayCallKind::kArrayFinalizeIntrinsic) { + if (arrayOp.getKind() == ArrayCallKind::kArrayFinalizeIntrinsic) { classifyUsesOfArray((ApplyInst *)arrayOp); continue; } diff --git a/lib/SILOptimizer/LoopTransforms/LICM.cpp b/lib/SILOptimizer/LoopTransforms/LICM.cpp deleted file mode 100644 index 567990e355560..0000000000000 --- a/lib/SILOptimizer/LoopTransforms/LICM.cpp +++ /dev/null @@ -1,1610 +0,0 @@ -//===--- LICM.cpp - Loop invariant code motion ----------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "sil-licm" - -#include "swift/Basic/Assertions.h" -#include "swift/SIL/Dominance.h" -#include "swift/SIL/InstructionUtils.h" -#include "swift/SIL/MemAccessUtils.h" -#include "swift/SIL/Projection.h" -#include "swift/SIL/SILArgument.h" -#include "swift/SIL/SILBuilder.h" -#include "swift/SIL/SILInstruction.h" -#include "swift/SIL/BasicBlockBits.h" -#include "swift/SILOptimizer/Analysis/AccessStorageAnalysis.h" -#include "swift/SILOptimizer/Analysis/AliasAnalysis.h" -#include "swift/SILOptimizer/Analysis/Analysis.h" -#include "swift/SILOptimizer/Analysis/ArraySemantic.h" -#include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h" -#include "swift/SILOptimizer/Analysis/DominanceAnalysis.h" -#include "swift/SILOptimizer/Analysis/LoopAnalysis.h" -#include "swift/SILOptimizer/PassManager/Passes.h" -#include "swift/SILOptimizer/PassManager/Transforms.h" -#include "swift/SILOptimizer/Utils/CFGOptUtils.h" -#include "swift/SILOptimizer/Utils/InstOptUtils.h" -#include "swift/SILOptimizer/Utils/OwnershipOptUtils.h" -#include "swift/SILOptimizer/Utils/SILSSAUpdater.h" - -#include "llvm/ADT/DepthFirstIterator.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/Support/Debug.h" - -#include "llvm/Support/CommandLine.h" - -using namespace swift; - -namespace { - -/// Instructions which can be hoisted: -/// loads, function calls without side effects and (some) exclusivity checks -using InstSet = llvm::SmallPtrSet; - -using InstVector = llvm::SmallVector; - -/// Returns true if the \p SideEffectInsts set contains any memory writes which -/// may alias with the memory addressed by \a LI. -template -static bool mayWriteTo(AliasAnalysis *AA, InstSet &SideEffectInsts, - UnaryInstructionBase *Inst) { - for (auto *I : SideEffectInsts) - if (AA->mayWriteToMemory(I, Inst->getOperand())) { - LLVM_DEBUG(llvm::dbgs() << " mayWriteTo\n" << *I << " to " - << *Inst << "\n"); - return true; - } - return false; -} - -/// Returns a non-null StoreInst if \p I is a store to \p accessPath. -static StoreInst *isStoreToAccess(SILInstruction *I, AccessPath accessPath) { - auto *SI = dyn_cast(I); - if (!SI) - return nullptr; - - // TODO: handle StoreOwnershipQualifier::Init - if (SI->getOwnershipQualifier() == StoreOwnershipQualifier::Init) - return nullptr; - - auto storeAccessPath = AccessPath::compute(SI->getDest()); - if (accessPath != storeAccessPath) - return nullptr; - - return SI; -} - -struct LoadWithAccess { - LoadInst *li = nullptr; - AccessPath accessPath; - - operator bool() { return li != nullptr; } -}; - -static LoadWithAccess doesLoadOverlapAccess(SILInstruction *I, - AccessPath accessPath) { - auto *LI = dyn_cast_or_null(I); - if (!LI) - return LoadWithAccess(); - - // TODO: handle LoadOwnershipQualifier::Take - if (LI->getOwnershipQualifier() == LoadOwnershipQualifier::Take) - return LoadWithAccess(); - - AccessPath loadAccessPath = AccessPath::compute(LI->getOperand()); - if (!loadAccessPath.isValid()) - return LoadWithAccess(); - - // Don't use AccessPath::mayOverlap. We only want definite overlap. - if (loadAccessPath.contains(accessPath) - || accessPath.contains(loadAccessPath)) { - return {LI, loadAccessPath}; - } - return LoadWithAccess(); -} - -/// Returns a valid LoadWithAccess if \p I is a load from \p accessPath or a -/// projected address from \p accessPath. -static LoadWithAccess isLoadWithinAccess(SILInstruction *I, - AccessPath accessPath) { - auto loadWithAccess = doesLoadOverlapAccess(I, accessPath); - if (!loadWithAccess) - return loadWithAccess; - - // Make sure that any additional path components beyond the store's access - // path can be converted to value projections during projectLoadValue (it - // currently only supports StructElementAddr and TupleElementAddr). - auto storePathNode = accessPath.getPathNode(); - auto loadPathNode = loadWithAccess.accessPath.getPathNode(); - SILValue loadAddr = loadWithAccess.li->getOperand(); - while (loadPathNode != storePathNode) { - if (!isa(loadAddr) - && !isa(loadAddr)) { - return LoadWithAccess(); - } - loadAddr = cast(loadAddr)->getOperand(0); - loadPathNode = loadPathNode.getParent(); - } - return loadWithAccess; -} - -/// Returns true if all instructions in \p SideEffectInsts which may alias with -/// \p access are either loads or stores from \p access. -/// -/// \p storeAddr is only needed for AliasAnalysis until we have an interface -/// that supports AccessPath. -static bool isOnlyLoadedAndStored(AliasAnalysis *AA, InstSet &SideEffectInsts, - ArrayRef Loads, - ArrayRef Stores, - SILValue storeAddr, AccessPath accessPath) { - for (auto *I : SideEffectInsts) { - // Pass the original address value until we can fix AA - if (AA->mayReadOrWriteMemory(I, storeAddr) - && !isStoreToAccess(I, accessPath) - && !isLoadWithinAccess(I, accessPath)) { - return false; - } - } - for (auto *LI : Loads) { - if (AA->mayReadFromMemory(LI, storeAddr) - && !doesLoadOverlapAccess(LI, accessPath)) - return false; - } - for (auto *SI : Stores) { - if (AA->mayWriteToMemory(SI, storeAddr) && !isStoreToAccess(SI, accessPath)) - return false; - } - return true; -} - -/// Returns true if the \p SideEffectInsts set contains any memory writes which -/// may alias with any memory which is read by \p AI. -/// Note: This function should only be called on a read-only apply! -static bool mayWriteTo(AliasAnalysis *AA, BasicCalleeAnalysis *BCA, - InstSet &SideEffectInsts, ApplyInst *AI) { - - if (BCA->getMemoryBehavior(FullApplySite::isa(AI), /*observeRetains*/true) == - MemoryBehavior::None) { - return false; - } - - // Check if the memory addressed by the argument may alias any writes. - for (auto *inst : SideEffectInsts) { - switch (inst->getKind()) { - case SILInstructionKind::StoreInst: { - auto *si = cast(inst); - if (si->getOwnershipQualifier() == StoreOwnershipQualifier::Assign) - return true; - if (AA->mayReadFromMemory(AI, si->getDest())) - return true; - break; - } - case SILInstructionKind::CopyAddrInst: { - auto *ca = cast(inst); - if (!ca->isInitializationOfDest()) - return true; - if (AA->mayReadFromMemory(AI, ca->getDest())) - return true; - break; - } - case SILInstructionKind::ApplyInst: - case SILInstructionKind::BeginApplyInst: - case SILInstructionKind::TryApplyInst: { - if (BCA->getMemoryBehavior(FullApplySite::isa(inst), /*observeRetains*/false) > - MemoryBehavior::MayRead) - return true; - break; - } - case SILInstructionKind::CondFailInst: - case SILInstructionKind::StrongRetainInst: - case SILInstructionKind::UnmanagedRetainValueInst: - case SILInstructionKind::RetainValueInst: - case SILInstructionKind::StrongRetainUnownedInst: - case SILInstructionKind::FixLifetimeInst: - case SILInstructionKind::KeyPathInst: - case SILInstructionKind::DeallocStackInst: - case SILInstructionKind::DeallocStackRefInst: - case SILInstructionKind::DeallocRefInst: - break; - default: - if (inst->mayWriteToMemory()) - return true; - break; - } - } - return false; -} - -/// Returns true if \p sideEffectInst cannot be reordered with a call to a -/// global initializer. -static bool mayConflictWithGlobalInit(AliasAnalysis *AA, - SILInstruction *sideEffectInst, SILInstruction *globalInitCall) { - if (auto *SI = dyn_cast(sideEffectInst)) { - return AA->mayReadOrWriteMemory(globalInitCall, SI->getDest()); - } - if (auto *LI = dyn_cast(sideEffectInst)) { - return AA->mayWriteToMemory(globalInitCall, LI->getOperand()); - } - if (isa(sideEffectInst)) - return false; - return true; -} - -/// Returns true if any of the instructions in \p sideEffectInsts which are -/// post-dominated by a call to a global initializer cannot be reordered with -/// the call. -static bool mayConflictWithGlobalInit(AliasAnalysis *AA, - InstSet &sideEffectInsts, - SILInstruction *globalInitCall, - SILBasicBlock *preHeader, PostDominanceInfo *PD) { - if (!PD->dominates(globalInitCall->getParent(), preHeader)) - return true; - - SILBasicBlock *globalInitBlock = globalInitCall->getParent(); - for (auto *seInst : sideEffectInsts) { - // Only check instructions in blocks which are "before" (i.e. post-dominated - // by) the block which contains the init-call. - // Instructions which are before the call in the same block have already - // been checked. - if (PD->properlyDominates(globalInitBlock, seInst->getParent())) { - if (mayConflictWithGlobalInit(AA, seInst, globalInitCall)) - return true; - } - } - return false; -} - -/// Returns true if any of the instructions in \p sideEffectInsts cannot be -/// reordered with a call to a global initializer (which is in the same basic -/// block). -static bool mayConflictWithGlobalInit(AliasAnalysis *AA, - ArrayRef sideEffectInsts, - SILInstruction *globalInitCall) { - for (auto *seInst : sideEffectInsts) { - assert(seInst->getParent() == globalInitCall->getParent()); - if (mayConflictWithGlobalInit(AA, seInst, globalInitCall)) - return true; - } - return false; -} - -// When Hoisting / Sinking, -// Don't descend into control-dependent code. -// Only traverse into basic blocks that dominate all exits. -static void getDominatingBlocks(SmallVectorImpl &domBlocks, - SILLoop *Loop, DominanceInfo *DT) { - auto HeaderBB = Loop->getHeader(); - auto DTRoot = DT->getNode(HeaderBB); - SmallVector ExitingAndLatchBBs; - Loop->getExitingAndLatchBlocks(ExitingAndLatchBBs); - for (llvm::df_iterator It = llvm::df_begin(DTRoot), - E = llvm::df_end(DTRoot); - It != E;) { - auto *CurBB = It->getBlock(); - - // Don't decent into control-dependent code. Only traverse into basic blocks - // that dominate all exits. - if (!std::all_of(ExitingAndLatchBBs.begin(), ExitingAndLatchBBs.end(), - [=](SILBasicBlock *ExitBB) { - return DT->dominates(CurBB, ExitBB); - })) { - LLVM_DEBUG(llvm::dbgs() << " skipping conditional block " - << *CurBB << "\n"); - It.skipChildren(); - continue; - } - domBlocks.push_back(CurBB); - // Next block in dominator tree. - ++It; - } -} - -/// Returns true if \p v is loop invariant in \p L. -static bool isLoopInvariant(SILValue v, SILLoop *L) { - if (SILBasicBlock *parent = v->getParentBlock()) - return !L->contains(parent); - return false; -} - -static bool hoistInstruction(DominanceInfo *DT, SILInstruction *Inst, - SILLoop *Loop, SILBasicBlock *&Preheader) { - auto Operands = Inst->getAllOperands(); - if (!std::all_of(Operands.begin(), Operands.end(), [=](Operand &Op) { - return isLoopInvariant(Op.get(), Loop); - })) { - LLVM_DEBUG(llvm::dbgs() << " loop variant operands\n"); - return false; - } - - auto mvBefore = Preheader->getTerminator(); - ArraySemanticsCall semCall(Inst); - if (semCall.canHoist(mvBefore, DT)) { - semCall.hoist(mvBefore, DT); - } else { - Inst->moveBefore(mvBefore); - } - return true; -} - -static bool hoistInstructions(SILLoop *Loop, DominanceInfo *DT, - InstSet &HoistUpSet) { - LLVM_DEBUG(llvm::dbgs() << " Hoisting instructions.\n"); - auto Preheader = Loop->getLoopPreheader(); - assert(Preheader && "Expected a preheader"); - bool Changed = false; - SmallVector domBlocks; - getDominatingBlocks(domBlocks, Loop, DT); - - for (auto *CurBB : domBlocks) { - // We know that the block is guaranteed to be executed. Hoist if we can. - for (auto InstIt = CurBB->begin(), E = CurBB->end(); InstIt != E;) { - SILInstruction *Inst = &*InstIt; - ++InstIt; - LLVM_DEBUG(llvm::dbgs() << " looking at " << *Inst); - if (!HoistUpSet.count(Inst)) { - continue; - } - if (!hoistInstruction(DT, Inst, Loop, Preheader)) { - continue; - } - LLVM_DEBUG(llvm::dbgs() << "Hoisted " << *Inst); - Changed = true; - } - } - - return Changed; -} - -/// Summary of side effect instructions occurring in the loop tree rooted at \p -/// Loop. This includes all writes of the sub loops and the loop itself. -struct LoopNestSummary { - SILLoop *Loop; - InstSet SideEffectInsts; - - LoopNestSummary(SILLoop *Curr) : Loop(Curr) {} - - - void copySummary(LoopNestSummary &Other) { - SideEffectInsts.insert(Other.SideEffectInsts.begin(), Other.SideEffectInsts.end()); - } - - LoopNestSummary(const LoopNestSummary &) = delete; - LoopNestSummary &operator=(const LoopNestSummary &) = delete; - LoopNestSummary(LoopNestSummary &&) = delete; -}; - -static unsigned getEdgeIndex(SILBasicBlock *BB, SILBasicBlock *ExitingBB) { - auto Succs = ExitingBB->getSuccessors(); - for (unsigned EdgeIdx = 0; EdgeIdx < Succs.size(); ++EdgeIdx) { - SILBasicBlock *CurrBB = Succs[EdgeIdx]; - if (CurrBB == BB) { - return EdgeIdx; - } - } - llvm_unreachable("BB is not a Successor"); -} - -static bool sinkInstruction(DominanceInfo *DT, - std::unique_ptr &LoopSummary, - SILInstruction *Inst, SILLoopInfo *LI) { - auto *Loop = LoopSummary->Loop; - SmallVector ExitBBs; - Loop->getExitBlocks(ExitBBs); - SmallVector NewExitBBs; - SmallVector ExitingBBs; - Loop->getExitingBlocks(ExitingBBs); - auto *ExitBB = Loop->getExitBlock(); - - bool Changed = false; - for (auto *ExitingBB : ExitingBBs) { - SmallVector BBSuccessors; - auto Succs = ExitingBB->getSuccessors(); - for (unsigned EdgeIdx = 0; EdgeIdx < Succs.size(); ++EdgeIdx) { - SILBasicBlock *BB = Succs[EdgeIdx]; - BBSuccessors.push_back(BB); - } - while (!BBSuccessors.empty()) { - SILBasicBlock *BB = BBSuccessors.pop_back_val(); - if (std::find(NewExitBBs.begin(), NewExitBBs.end(), BB) != - NewExitBBs.end()) { - // Already got a copy there - continue; - } - auto EdgeIdx = getEdgeIndex(BB, ExitingBB); - SILBasicBlock *OutsideBB = nullptr; - if (std::find(ExitBBs.begin(), ExitBBs.end(), BB) != ExitBBs.end()) { - auto *SplitBB = - splitCriticalEdge(ExitingBB->getTerminator(), EdgeIdx, DT, LI); - OutsideBB = SplitBB ? SplitBB : BB; - NewExitBBs.push_back(OutsideBB); - } - if (!OutsideBB) { - continue; - } - // If OutsideBB already contains Inst -> skip - // This might happen if we have a conditional control flow - // And a pair - // We hoisted the first part, we can safely ignore sinking - auto matchPred = [&](SILInstruction &CurrIns) { - return Inst->isIdenticalTo(&CurrIns); - }; - if (std::find_if(OutsideBB->begin(), OutsideBB->end(), matchPred) != - OutsideBB->end()) { - LLVM_DEBUG(llvm::errs() << " instruction already at exit BB " - << *Inst); - ExitBB = nullptr; - } else if (ExitBB) { - // easy case - LLVM_DEBUG(llvm::errs() << " moving instruction to exit BB " << *Inst); - Inst->moveBefore(&*OutsideBB->begin()); - } else { - LLVM_DEBUG(llvm::errs() << " cloning instruction to exit BB " - << *Inst); - Inst->clone(&*OutsideBB->begin()); - } - Changed = true; - } - } - if (Changed && !ExitBB) { - // Created clones of instruction - // Remove it from the side-effect set - dangling pointer - LoopSummary->SideEffectInsts.erase(Inst); - Inst->getParent()->erase(Inst); - } - return Changed; -} - -static bool sinkInstructions(std::unique_ptr &LoopSummary, - DominanceInfo *DT, SILLoopInfo *LI, - InstVector &SinkDownSet) { - auto *Loop = LoopSummary->Loop; - LLVM_DEBUG(llvm::errs() << " Sink instructions attempt\n"); - SmallVector domBlocks; - getDominatingBlocks(domBlocks, Loop, DT); - - bool Changed = false; - for (auto *Inst : SinkDownSet) { - // only sink if the block is guaranteed to be executed. - if (std::find(domBlocks.begin(), domBlocks.end(), Inst->getParent()) == - domBlocks.end()) { - continue; - } - Changed |= sinkInstruction(DT, LoopSummary, Inst, LI); - } - - return Changed; -} - -static void getEndAccesses(BeginAccessInst *BI, - SmallVectorImpl &EndAccesses) { - for (auto Use : BI->getUses()) { - auto *User = Use->getUser(); - auto *EI = dyn_cast(User); - if (!EI) { - continue; - } - EndAccesses.push_back(EI); - } -} - -static bool -hoistSpecialInstruction(std::unique_ptr &LoopSummary, - DominanceInfo *DT, SILLoopInfo *LI, InstVector &Special) { - auto *Loop = LoopSummary->Loop; - LLVM_DEBUG(llvm::errs() << " Hoist and Sink pairs attempt\n"); - auto Preheader = Loop->getLoopPreheader(); - assert(Preheader && "Expected a preheader"); - - bool Changed = false; - - for (auto *Inst : Special) { - if (isa(Inst) && LoopSummary->Loop->hasNoExitBlocks()) { - // If no exit block, don't try to hoist BeginAccess because - // sinking EndAccess would fail later. - continue; - } - if (!hoistInstruction(DT, Inst, Loop, Preheader)) { - continue; - } - if (auto *BI = dyn_cast(Inst)) { - SmallVector Ends; - getEndAccesses(BI, Ends); - LLVM_DEBUG(llvm::dbgs() << "Hoisted BeginAccess " << *BI); - for (auto *instSink : Ends) { - if (!sinkInstruction(DT, LoopSummary, instSink, LI)) { - llvm_unreachable("LICM: Could not perform must-sink instruction"); - } - } - LLVM_DEBUG(llvm::errs() << " Successfully hoisted and sank pair\n"); - } else { - LLVM_DEBUG(llvm::dbgs() << "Hoisted RefElementAddr " - << *static_cast(Inst)); - } - Changed = true; - } - - return Changed; -} - -/// Optimize the loop tree bottom up propagating loop's summaries up the -/// loop tree. -class LoopTreeOptimization { - llvm::DenseMap> - LoopNestSummaryMap; - SmallVector BotUpWorkList; - InstSet toDelete; - SILLoopInfo *LoopInfo; - AliasAnalysis *AA; - BasicCalleeAnalysis *BCA; - DominanceInfo *DomTree; - PostDominanceAnalysis *PDA; - PostDominanceInfo *postDomTree = nullptr; - AccessStorageAnalysis *ASA; - bool Changed; - - /// True if LICM is done on high-level SIL, i.e. semantic calls are not - /// inlined yet. In this case some semantic calls can be hoisted. - bool RunsOnHighLevelSIL; - - /// Instructions that we may be able to hoist up - InstSet HoistUp; - - /// Instructions that we may be able to sink down - InstVector SinkDown; - - /// Load and store instructions that we may be able to move out of the loop. - /// All loads and stores within a block must be in instruction order to - /// simplify replacement of values after SSA update. - InstVector LoadsAndStores; - - /// All access paths of the \p LoadsAndStores instructions. - llvm::SetVector LoadAndStoreAddrs; - - /// Hoistable Instructions that need special treatment - /// e.g. begin_access - InstVector SpecialHoist; - -public: - LoopTreeOptimization(SILLoop *TopLevelLoop, SILLoopInfo *LI, - AliasAnalysis *AA, BasicCalleeAnalysis *BCA, - DominanceInfo *DT, PostDominanceAnalysis *PDA, - AccessStorageAnalysis *ASA, - bool RunsOnHighLevelSil) - : LoopInfo(LI), AA(AA), BCA(BCA), DomTree(DT), PDA(PDA), ASA(ASA), - Changed(false), RunsOnHighLevelSIL(RunsOnHighLevelSil) { - // Collect loops for a recursive bottom-up traversal in the loop tree. - BotUpWorkList.push_back(TopLevelLoop); - for (unsigned i = 0; i < BotUpWorkList.size(); ++i) { - auto *L = BotUpWorkList[i]; - for (auto *SubLoop : *L) - BotUpWorkList.push_back(SubLoop); - } - } - - /// Optimize this loop tree. - bool optimize(); - -protected: - /// Propagate the sub-loops' summaries up to the current loop. - void propagateSummaries(std::unique_ptr &CurrSummary); - - bool isSafeReadOnlyApply(BasicCalleeAnalysis *BCA, ApplyInst *AI); - - /// Collect a set of instructions that can be hoisted - void analyzeCurrentLoop(std::unique_ptr &CurrSummary); - - SingleValueInstruction *splitLoad(SILValue splitAddress, - ArrayRef remainingPath, - SILBuilder &builder, - SmallVectorImpl &Loads, - unsigned ldStIdx); - - /// Given an \p accessPath that is only loaded and stored, split loads that - /// are wider than \p accessPath. - bool splitLoads(SmallVectorImpl &Loads, AccessPath accessPath, - SILValue storeAddr); - - /// Optimize the current loop nest. - bool optimizeLoop(std::unique_ptr &CurrSummary); - - /// Move all loads and stores from/to \p accessPath out of the \p loop. - void hoistLoadsAndStores(AccessPath accessPath, SILLoop *loop); - - /// Move all loads and stores from all addresses in LoadAndStoreAddrs out of - /// the \p loop. - /// - /// This is a combination of load hoisting and store sinking, e.g. - /// \code - /// preheader: - /// br header_block - /// header_block: - /// %x = load %not_aliased_addr - /// // use %x and define %y - /// store %y to %not_aliased_addr - /// ... - /// exit_block: - /// \endcode - /// is transformed to: - /// \code - /// preheader: - /// %x = load %not_aliased_addr - /// br header_block - /// header_block: - /// // use %x and define %y - /// ... - /// exit_block: - /// store %y to %not_aliased_addr - /// \endcode - bool hoistAllLoadsAndStores(SILLoop *loop); -}; -} // end anonymous namespace - -bool LoopTreeOptimization::optimize() { - // Process loops bottom up in the loop tree. - while (!BotUpWorkList.empty()) { - SILLoop *CurrentLoop = BotUpWorkList.pop_back_val(); - LLVM_DEBUG(llvm::dbgs() << "Processing loop " << *CurrentLoop); - - // Collect all summary of all sub loops of the current loop. Since we - // process the loop tree bottom up they are guaranteed to be available in - // the map. - auto CurrLoopSummary = std::make_unique(CurrentLoop); - propagateSummaries(CurrLoopSummary); - - // If the current loop changed, then we might reveal more instr to hoist - // For example, a fix_lifetime's operand, if hoisted outside, - // Might allow us to sink the instruction out of the loop - bool currChanged = false; - do { - // Analyze the current loop for instructions that can be hoisted. - analyzeCurrentLoop(CurrLoopSummary); - - currChanged = optimizeLoop(CurrLoopSummary); - if (currChanged) { - CurrLoopSummary->SideEffectInsts.clear(); - Changed = true; - } - - // Reset the data structures for next loop in the list - HoistUp.clear(); - SinkDown.clear(); - SpecialHoist.clear(); - } while (currChanged); - - // Store the summary for parent loops to use. - LoopNestSummaryMap[CurrentLoop] = std::move(CurrLoopSummary); - } - return Changed; -} - -void LoopTreeOptimization::propagateSummaries( - std::unique_ptr &CurrSummary) { - for (auto *SubLoop : *CurrSummary->Loop) { - assert(LoopNestSummaryMap.count(SubLoop) && "Must have data for sub loops"); - CurrSummary->copySummary(*LoopNestSummaryMap[SubLoop]); - LoopNestSummaryMap.erase(SubLoop); - } -} - -bool LoopTreeOptimization::isSafeReadOnlyApply(BasicCalleeAnalysis *BCA, ApplyInst *AI) { - if (auto ri = AI->getSingleResult()) { - // We don't balance CSE'd apply results which return an owned value. - if (ri.value().getConvention() != ResultConvention::Unowned) - return false; - } - - if (RunsOnHighLevelSIL) { - // The array-property-opt needs this semantic call inside the loop. - // After high-level SIL we can hoist it (if it's not inlined already). - if (ArraySemanticsCall(AI, "array.props.isNativeTypeChecked")) - return false; - } - - return BCA->getMemoryBehavior(AI, /*observeRetains*/false) <= - MemoryBehavior::MayRead; -} - -static void checkSideEffects(swift::SILInstruction &Inst, - InstSet &SideEffectInsts, - SmallVectorImpl &sideEffectsInBlock, - bool &hasOtherMemReadingInsts) { - if (Inst.mayHaveSideEffects()) { - SideEffectInsts.insert(&Inst); - sideEffectsInBlock.push_back(&Inst); - } else if (Inst.mayReadFromMemory()) { - hasOtherMemReadingInsts = true; - } -} - -/// Returns true if the \p Inst follows the default hoisting heuristic -static bool canHoistUpDefault(SILInstruction *inst, SILLoop *Loop, - DominanceInfo *DT, bool RunsOnHighLevelSil) { - auto Preheader = Loop->getLoopPreheader(); - if (!Preheader) { - return false; - } - - if (isa(inst) || isa(inst) || - isa(inst)) { - return false; - } - - // We can’t hoist everything that is hoist-able - // The canHoist method does not do all the required analysis - // Some of the work is done at COW Array Opt - // TODO: Refactor COW Array Opt + canHoist - radar 41601468 - ArraySemanticsCall semCall(inst); - switch (semCall.getKind()) { - case ArrayCallKind::kGetCount: - case ArrayCallKind::kGetCapacity: - if (RunsOnHighLevelSil && semCall.canHoist(Preheader->getTerminator(), DT)) - return true; - break; - case ArrayCallKind::kArrayPropsIsNativeTypeChecked: - // The array-property-opt needs this semantic call inside the loop. - // After high-level SIL we can hoist it (if it's not inlined already). - if (RunsOnHighLevelSil) - return false; - break; - default: - break; - } - - if (inst->getMemoryBehavior() == MemoryBehavior::None) { - return true; - } - return false; -} - -// Check If all the end accesses of the given begin do not prevent hoisting -// There are only two legal placements for the end access instructions: -// 1) Inside the same loop (sink to loop exists) -// Potential TODO: At loop exit block -static bool handledEndAccesses(BeginAccessInst *BI, SILLoop *Loop) { - SmallVector AllEnds; - getEndAccesses(BI, AllEnds); - if (AllEnds.empty()) { - return false; - } - for (auto *User : AllEnds) { - auto *BB = User->getParent(); - if (Loop->getBlocksSet().count(BB) != 0) { - continue; - } - return false; - } - return true; -} - -static bool isCoveredByScope(BeginAccessInst *BI, DominanceInfo *DT, - SILInstruction *applyInstr) { - if (!DT->dominates(BI, applyInstr)) - return false; - for (auto *EI : BI->getEndAccesses()) { - if (!DT->dominates(applyInstr, EI)) - return false; - } - return true; -} - -static bool analyzeBeginAccess(BeginAccessInst *BI, - SmallVector &BeginAccesses, - SmallVector &fullApplies, - InstSet &SideEffectInsts, - AccessStorageAnalysis *ASA, - DominanceInfo *DT) { - auto storage = AccessStorage::compute(BI->getSource()); - if (!storage) { - return false; - } - - auto BIAccessStorageNonNested = AccessStorage::compute(BI); - auto safeBeginPred = [&](BeginAccessInst *OtherBI) { - if (BI == OtherBI) { - return true; - } - return BIAccessStorageNonNested.isDistinctFrom( - AccessStorage::compute(OtherBI)); - }; - - if (!std::all_of(BeginAccesses.begin(), BeginAccesses.end(), safeBeginPred)) - return false; - - for (auto fullApply : fullApplies) { - FunctionAccessStorage callSiteAccesses; - ASA->getCallSiteEffects(callSiteAccesses, fullApply); - SILAccessKind accessKind = BI->getAccessKind(); - if (!callSiteAccesses.mayConflictWith(accessKind, storage)) - continue; - // Check if we can ignore this conflict: - // If the apply is “sandwiched” between the begin and end access, - // there’s no reason we can’t hoist out of the loop. - auto *applyInstr = fullApply.getInstruction(); - if (!isCoveredByScope(BI, DT, applyInstr)) - return false; - } - - // Check may releases - // Only class and global access that may alias would conflict - const AccessStorage::Kind kind = storage.getKind(); - if (kind != AccessStorage::Class && kind != AccessStorage::Global) { - return true; - } - // TODO Introduce "Pure Swift" deinitializers - // We can then make use of alias information for instr's operands - // If they don't alias - we might get away with not recording a conflict - for (SILInstruction *I : SideEffectInsts) { - // we actually compute all SideEffectInsts in analyzeCurrentLoop - if (!I->mayRelease()) { - continue; - } - if (!isCoveredByScope(BI, DT, I)) - return false; - } - - return true; -} - -// Analyzes current loop for hosting/sinking potential: -// Computes set of instructions we may be able to move out of the loop -// Important Note: -// We can't bail out of this method! we have to run it on all loops. -// We *need* to discover all SideEffectInsts - -// even if the loop is otherwise skipped! -// This is because outer loops will depend on the inner loop's writes. -// -// This may split some loads into smaller loads. -void LoopTreeOptimization::analyzeCurrentLoop( - std::unique_ptr &CurrSummary) { - InstSet &sideEffects = CurrSummary->SideEffectInsts; - SILLoop *Loop = CurrSummary->Loop; - LLVM_DEBUG(llvm::dbgs() << " Analyzing accesses.\n"); - - auto *Preheader = Loop->getLoopPreheader(); - if (!Preheader) { - // Can't hoist/sink instructions - return; - } - - // Interesting instructions in the loop: - SmallVector ReadOnlyApplies; - - // Contains either: - // * an apply to the addressor of the global - // * a builtin "once" of the global initializer - SmallVector globalInitCalls; - - SmallVector Loads; - SmallVector Stores; - SmallVector FixLifetimes; - SmallVector BeginAccesses; - SmallVector fullApplies; - - // True if the loop has instructions which (may) read from memory, which are not - // in `Loads` and not in `sideEffects`. - bool hasOtherMemReadingInsts = false; - - for (auto *BB : Loop->getBlocks()) { - SmallVector sideEffectsInBlock; - for (auto &Inst : *BB) { - if (hasOwnershipOperandsOrResults(&Inst)) { - checkSideEffects(Inst, sideEffects, sideEffectsInBlock, hasOtherMemReadingInsts); - // Collect fullApplies to be checked in analyzeBeginAccess - if (auto fullApply = FullApplySite::isa(&Inst)) { - fullApplies.push_back(fullApply); - } - continue; - } - switch (Inst.getKind()) { - case SILInstructionKind::FixLifetimeInst: { - auto *FL = cast(&Inst); - if (DomTree->dominates(FL->getOperand()->getParentBlock(), Preheader)) - FixLifetimes.push_back(FL); - // We can ignore the side effects of FixLifetimes - break; - } - case SILInstructionKind::LoadInst: - Loads.push_back(cast(&Inst)); - LoadsAndStores.push_back(&Inst); - break; - case SILInstructionKind::StoreInst: { - auto *store = cast(&Inst); - switch (store->getOwnershipQualifier()) { - case StoreOwnershipQualifier::Assign: - case StoreOwnershipQualifier::Init: - // Currently not supported. - continue; - case StoreOwnershipQualifier::Unqualified: - case StoreOwnershipQualifier::Trivial: - break; - } - Stores.push_back(store); - LoadsAndStores.push_back(&Inst); - checkSideEffects(Inst, sideEffects, sideEffectsInBlock, hasOtherMemReadingInsts); - break; - } - case SILInstructionKind::BeginAccessInst: - BeginAccesses.push_back(cast(&Inst)); - checkSideEffects(Inst, sideEffects, sideEffectsInBlock, hasOtherMemReadingInsts); - break; - case SILInstructionKind::RefElementAddrInst: - SpecialHoist.push_back(cast(&Inst)); - break; - case swift::SILInstructionKind::CondFailInst: - // We can (and must) hoist cond_fail instructions if the operand is - // invariant. We must hoist them so that we preserve memory safety. A - // cond_fail that would have protected (executed before) a memory access - // must - after hoisting - also be executed before said access. - HoistUp.insert(&Inst); - checkSideEffects(Inst, sideEffects, sideEffectsInBlock, hasOtherMemReadingInsts); - break; - case SILInstructionKind::ApplyInst: { - auto *AI = cast(&Inst); - if (isSafeReadOnlyApply(BCA, AI)) { - ReadOnlyApplies.push_back(AI); - } else if (SILFunction *callee = AI->getReferencedFunctionOrNull()) { - // Calls to global inits are different because we don't care about - // side effects which are "after" the call in the loop. - if (callee->isGlobalInit() && - // Check against side-effects within the same block. - // Side-effects in other blocks are checked later (after we - // scanned all blocks of the loop). - !mayConflictWithGlobalInit(AA, sideEffectsInBlock, &Inst)) - globalInitCalls.push_back(&Inst); - } - // check for array semantics and side effects - same as default - LLVM_FALLTHROUGH; - } - default: - if (auto fullApply = FullApplySite::isa(&Inst)) { - fullApplies.push_back(fullApply); - } else if (auto *bi = dyn_cast(&Inst)) { - switch (bi->getBuiltinInfo().ID) { - case BuiltinValueKind::Once: - case BuiltinValueKind::OnceWithContext: - if (!mayConflictWithGlobalInit(AA, sideEffectsInBlock, &Inst)) - globalInitCalls.push_back(&Inst); - break; - default: - break; - } - } - - checkSideEffects(Inst, sideEffects, sideEffectsInBlock, hasOtherMemReadingInsts); - if (canHoistUpDefault(&Inst, Loop, DomTree, RunsOnHighLevelSIL)) { - HoistUp.insert(&Inst); - } - break; - } - } - } - - // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. - if (ReadOnlyApplies.size() * sideEffects.size() < 8000) { - for (auto *AI : ReadOnlyApplies) { - if (!mayWriteTo(AA, BCA, sideEffects, AI)) { - HoistUp.insert(AI); - } - } - } - // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. - if (Loads.size() * sideEffects.size() < 8000) { - for (auto *LI : Loads) { - if (!mayWriteTo(AA, sideEffects, LI)) { - HoistUp.insert(LI); - } - } - } - - if (!globalInitCalls.empty()) { - if (!postDomTree) { - postDomTree = PDA->get(Preheader->getParent()); - } - if (postDomTree->getRootNode()) { - for (SILInstruction *ginitCall : globalInitCalls) { - // Check against side effects which are "before" (i.e. post-dominated - // by) the global initializer call. - if (!mayConflictWithGlobalInit(AA, sideEffects, ginitCall, Preheader, - postDomTree)) { - HoistUp.insert(ginitCall); - } - } - } - } - - if (!hasOtherMemReadingInsts) { - // Collect memory locations for which we can move all loads and stores out - // of the loop. - // - // Note: The Loads set and LoadsAndStores set may mutate during this loop. - for (StoreInst *SI : Stores) { - // Use AccessPathWithBase to recover a base address that can be used for - // newly inserted memory operations. If we instead teach hoistLoadsAndStores - // how to rematerialize global_addr, then we don't need this base. - auto access = AccessPathWithBase::compute(SI->getDest()); - auto accessPath = access.accessPath; - if (accessPath.isValid() && - (access.base && isLoopInvariant(access.base, Loop))) { - if (isOnlyLoadedAndStored(AA, sideEffects, Loads, Stores, SI->getDest(), - accessPath)) { - if (!LoadAndStoreAddrs.count(accessPath)) { - if (splitLoads(Loads, accessPath, SI->getDest())) { - LoadAndStoreAddrs.insert(accessPath); - } - } - } - } - } - } - if (!FixLifetimes.empty()) { - bool sideEffectsMayRelease = - std::any_of(sideEffects.begin(), sideEffects.end(), - [&](SILInstruction *W) { return W->mayRelease(); }); - for (auto *FL : FixLifetimes) { - if (!FL->getOperand()->getType().isAddress()) - continue; - if (!sideEffectsMayRelease || !mayWriteTo(AA, sideEffects, FL)) { - SinkDown.push_back(FL); - } - } - } - for (auto *BI : BeginAccesses) { - if (!handledEndAccesses(BI, Loop)) { - LLVM_DEBUG(llvm::dbgs() << "Skipping: " << *BI); - LLVM_DEBUG(llvm::dbgs() << "Some end accesses can't be handled\n"); - continue; - } - if (analyzeBeginAccess(BI, BeginAccesses, fullApplies, sideEffects, ASA, - DomTree)) { - SpecialHoist.push_back(BI); - } - } -} - -// Recursively determine whether the innerAddress is a direct tuple or struct -// projection chain from outerPath. Populate \p reversePathIndices with the path -// difference. -static bool -computeInnerAccessPath(AccessPath::PathNode outerPath, - AccessPath::PathNode innerPath, SILValue innerAddress, - SmallVectorImpl &reversePathIndices) { - if (outerPath == innerPath) - return true; - - auto *sea = dyn_cast(innerAddress); - - if (sea && sea->getStructDecl()->hasUnreferenceableStorage()) { - return false; - } - - if (!sea && !isa(innerAddress)) { - return false; - } - assert(ProjectionIndex(innerAddress).Index - == innerPath.getIndex().getSubObjectIndex()); - - reversePathIndices.push_back(innerPath.getIndex()); - SILValue srcAddr = cast(innerAddress)->getOperand(0); - if (!computeInnerAccessPath(outerPath, innerPath.getParent(), srcAddr, - reversePathIndices)) { - return false; - } - return true; -} - -/// Split a load from \p outerAddress recursively following remainingPath. -/// -/// Creates a load with identical \p accessPath and a set of -/// non-overlapping loads. Add the new non-overlapping loads to HoistUp. -/// -/// \p ldstIdx is the index into LoadsAndStores of the original outer load. -/// -/// Return the aggregate produced by merging the loads. -SingleValueInstruction *LoopTreeOptimization::splitLoad( - SILValue splitAddress, ArrayRef remainingPath, - SILBuilder &builder, SmallVectorImpl &Loads, unsigned ldstIdx) { - auto loc = LoadsAndStores[ldstIdx]->getLoc(); - LoadOwnershipQualifier ownership = builder.getFunction().hasOwnership() ? - LoadOwnershipQualifier::Trivial : - LoadOwnershipQualifier::Unqualified; - - // Recurse until we have a load that matches accessPath. - if (remainingPath.empty()) { - // Create a load that matches the stored access path. - LoadInst *load = builder.createLoad(loc, splitAddress, ownership); - Loads.push_back(load); - // Replace the outer load in the list of loads and stores to hoist and - // sink. LoadsAndStores must remain in instruction order. - LoadsAndStores[ldstIdx] = load; - LLVM_DEBUG(llvm::dbgs() << "Created load from stored path: " << *load); - return load; - } - auto recordDisjointLoad = [&](LoadInst *newLoad) { - Loads.push_back(newLoad); - LoadsAndStores.insert(LoadsAndStores.begin() + ldstIdx + 1, newLoad); - }; - auto subIndex = remainingPath.back().getSubObjectIndex(); - SILType loadTy = splitAddress->getType(); - if (CanTupleType tupleTy = loadTy.getAs()) { - SmallVector elements; - for (int tupleIdx : range(tupleTy->getNumElements())) { - auto *projection = builder.createTupleElementAddr( - loc, splitAddress, tupleIdx, loadTy.getTupleElementType(tupleIdx)); - SILValue elementVal; - if (tupleIdx == subIndex) { - elementVal = splitLoad(projection, remainingPath.drop_back(), builder, - Loads, ldstIdx); - } else { - elementVal = builder.createLoad(loc, projection, ownership); - recordDisjointLoad(cast(elementVal)); - } - elements.push_back(elementVal); - } - return builder.createTuple(loc, loadTy.getObjectType(), elements); - } - auto structTy = loadTy.getStructOrBoundGenericStruct(); - assert(structTy && "tuple and struct elements are checked earlier"); - auto &module = builder.getModule(); - auto expansionContext = builder.getFunction().getTypeExpansionContext(); - - SmallVector elements; - int fieldIdx = 0; - for (auto *field : structTy->getStoredProperties()) { - SILType fieldTy = loadTy.getFieldType(field, module, expansionContext); - auto *projection = - builder.createStructElementAddr(loc, splitAddress, field, fieldTy); - SILValue fieldVal; - if (fieldIdx++ == subIndex) - fieldVal = splitLoad(projection, remainingPath.drop_back(), builder, - Loads, ldstIdx); - else { - fieldVal = builder.createLoad(loc, projection, ownership); - recordDisjointLoad(cast(fieldVal)); - } - elements.push_back(fieldVal); - } - return builder.createStruct(loc, loadTy.getObjectType(), elements); -} - -/// Find all loads that contain \p accessPath. Split them into a load with -/// identical accessPath and a set of non-overlapping loads. Add the new -/// non-overlapping loads to LoadsAndStores and HoistUp. -/// -/// TODO: The \p storeAddr parameter is only needed until we have an -/// AliasAnalysis interface that handles AccessPath. -bool LoopTreeOptimization::splitLoads(SmallVectorImpl &Loads, - AccessPath accessPath, - SILValue storeAddr) { - // The Loads set may mutate during this loop, but we only want to visit the - // original set. - for (unsigned loadsIdx = 0, endIdx = Loads.size(); loadsIdx != endIdx; - ++loadsIdx) { - auto *load = Loads[loadsIdx]; - if (toDelete.count(load)) - continue; - - if (!AA->mayReadFromMemory(load, storeAddr)) - continue; - - AccessPath loadAccessPath = AccessPath::compute(load->getOperand()); - if (accessPath.contains(loadAccessPath)) - continue; - - assert(loadAccessPath.contains(accessPath)); - LLVM_DEBUG(llvm::dbgs() << "Overlaps with loop stores: " << *load); - SmallVector reversePathIndices; - if (!computeInnerAccessPath(loadAccessPath.getPathNode(), - accessPath.getPathNode(), storeAddr, - reversePathIndices)) { - return false; - } - // Found a load wider than the store to accessPath. - // - // SplitLoads is called for each unique access path in the loop that is - // only loaded from and stored to and this loop takes time proportional to: - // num-wide-loads x num-fields x num-loop-memops - // - // For each load wider than the store, it creates a new load for each field - // in that type. Each new load is inserted in the LoadsAndStores vector. To - // avoid super-linear behavior for large types (e.g. giant tuples), limit - // growth of new loads to an arbitrary constant factor per access path. - if (Loads.size() >= endIdx + 6) { - LLVM_DEBUG(llvm::dbgs() << "...Refusing to split more loads\n"); - return false; - } - LLVM_DEBUG(llvm::dbgs() << "...Splitting load\n"); - - unsigned ldstIdx = [this, load]() { - auto ldstIter = llvm::find(LoadsAndStores, load); - assert(ldstIter != LoadsAndStores.end() && "outerLoad missing"); - return std::distance(LoadsAndStores.begin(), ldstIter); - }(); - - SILBuilderWithScope builder(load); - - SILValue aggregateVal = splitLoad(load->getOperand(), reversePathIndices, - builder, Loads, ldstIdx); - - load->replaceAllUsesWith(aggregateVal); - auto iterAndInserted = toDelete.insert(load); - (void)iterAndInserted; - assert(iterAndInserted.second && "the same load should only be split once"); - } - return true; -} - -bool LoopTreeOptimization::optimizeLoop( - std::unique_ptr &CurrSummary) { - auto *CurrentLoop = CurrSummary->Loop; - // We only support Loops with a preheader - if (!CurrentLoop->getLoopPreheader()) - return false; - bool currChanged = false; - if (hoistAllLoadsAndStores(CurrentLoop)) - return true; - - currChanged |= hoistInstructions(CurrentLoop, DomTree, HoistUp); - currChanged |= sinkInstructions(CurrSummary, DomTree, LoopInfo, SinkDown); - currChanged |= - hoistSpecialInstruction(CurrSummary, DomTree, LoopInfo, SpecialHoist); - - assert(toDelete.empty() && "only hostAllLoadsAndStores deletes"); - return currChanged; -} - -/// Creates a value projection from \p rootVal based on the address projection -/// from \a rootVal to \a addr. -static SILValue projectLoadValue(SILValue addr, AccessPath accessPath, - SILValue rootVal, AccessPath rootAccessPath, - SILInstruction *beforeInst) { - if (accessPath == rootAccessPath) - return rootVal; - - auto pathNode = accessPath.getPathNode(); - int elementIdx = pathNode.getIndex().getSubObjectIndex(); - if (auto *SEI = dyn_cast(addr)) { - assert(ProjectionIndex(SEI).Index == elementIdx); - SILValue val = projectLoadValue( - SEI->getOperand(), - AccessPath(accessPath.getStorage(), pathNode.getParent(), - accessPath.getOffset()), - rootVal, rootAccessPath, beforeInst); - SILBuilder B(beforeInst); - return B.createStructExtract(beforeInst->getLoc(), val, SEI->getField(), - SEI->getType().getObjectType()); - } - if (auto *TEI = dyn_cast(addr)) { - assert(ProjectionIndex(TEI).Index == elementIdx); - SILValue val = projectLoadValue( - TEI->getOperand(), - AccessPath(accessPath.getStorage(), pathNode.getParent(), - accessPath.getOffset()), - rootVal, rootAccessPath, beforeInst); - SILBuilder B(beforeInst); - return B.createTupleExtract(beforeInst->getLoc(), val, TEI->getFieldIndex(), - TEI->getType().getObjectType()); - } - llvm_unreachable("unknown projection"); -} - -/// Returns true if all stores to \p addr commonly dominate the loop exits. -static bool -storesCommonlyDominateLoopExits(AccessPath accessPath, - SILLoop *loop, - ArrayRef exitingBlocks) { - BasicBlockSet stores(loop->getHeader()->getParent()); - SmallVector uses; - // Collect as many recognizable stores as possible. It's ok if not all stores - // are collected. - accessPath.collectUses(uses, AccessUseType::Exact, loop->getFunction()); - for (Operand *use : uses) { - SILInstruction *user = use->getUser(); - if (isa(user)) - stores.insert(user->getParent()); - } - SILBasicBlock *header = loop->getHeader(); - // If a store is in the loop header, we already know that it's dominating all - // loop exits. - if (stores.contains(header)) - return true; - - // Also a store in the pre-header dominates all exists. Although the situation - // is a bit different here: the store in the pre-header remains - it's not - // (re)moved by the LICM transformation. - // But even if the loop-stores are not dominating the loop exits, it - // makes sense to move them out of the loop if this case. When this is done, - // dead-store-elimination can then most likely eliminate the store in the - // pre-header. - // - // pre_header: - // store %v1 to %addr - // header: - // cond_br %cond, then, tail - // then: - // store %v2 to %addr // a conditional store in the loop - // br tail - // tail: - // cond_br %loop_cond, header, exit - // exit: - // - // will be transformed to - // - // pre_header: - // store %v1 to %addr // <- can be removed by DSE afterwards - // header: - // cond_br %cond, then, tail - // then: - // br tail - // tail(%phi): - // cond_br %loop_cond, header, exit - // exit: - // store %phi to %addr - // - if (stores.contains(loop->getLoopPreheader())) - return true; - - // Propagate the store-is-not-alive flag through the control flow in the loop, - // starting at the header. - BasicBlockSet storesNotAlive(loop->getHeader()->getParent()); - storesNotAlive.insert(header); - bool changed = false; - do { - changed = false; - for (SILBasicBlock *block : loop->blocks()) { - bool storeAlive = !storesNotAlive.contains(block); - if (storeAlive && !stores.contains(block) && - std::any_of(block->pred_begin(), block->pred_end(), - [&](SILBasicBlock *b) { return storesNotAlive.contains(b); })) { - storesNotAlive.insert(block); - changed = true; - } - } - } while (changed); - - auto isUnreachableBlock = [](SILBasicBlock *succ) { - return isa(succ->getTerminator()); - }; - - // Check if the store-is-not-alive flag reaches any of the exits. - for (SILBasicBlock *eb : exitingBlocks) { - // Ignore loop exits to blocks which end in an unreachable. - if (!std::any_of(eb->succ_begin(), eb->succ_end(), isUnreachableBlock) && - storesNotAlive.contains(eb)) { - return false; - } - } - return true; -} - -void LoopTreeOptimization:: -hoistLoadsAndStores(AccessPath accessPath, SILLoop *loop) { - SmallVector exitingAndLatchBlocks; - loop->getExitingAndLatchBlocks(exitingAndLatchBlocks); - - // This is not a requirement for functional correctness, but we don't want to - // _speculatively_ load and store the value (outside of the loop). - if (!storesCommonlyDominateLoopExits(accessPath, loop, - exitingAndLatchBlocks)) - return; - - // Inserting the stores requires the exit edges to be not critical. - for (SILBasicBlock *exitingOrLatchBlock : exitingAndLatchBlocks) { - for (unsigned idx = 0, e = exitingOrLatchBlock->getSuccessors().size(); - idx != e; ++idx) { - // exitingBlock->getSuccessors() must not be moved out of this loop, - // because the successor list is invalidated by splitCriticalEdge. - if (!loop->contains(exitingOrLatchBlock->getSuccessors()[idx])) { - splitCriticalEdge(exitingOrLatchBlock->getTerminator(), idx, DomTree, - LoopInfo); - } - } - } - - SILBasicBlock *preheader = loop->getLoopPreheader(); - assert(preheader && "Expected a preheader"); - - // Initially load the value in the loop pre header. - SILBuilder B(preheader->getTerminator()); - SILValue storeAddr; - SILSSAUpdater ssaUpdater; - - // Set all stored values as available values in the ssaUpdater. - // If there are multiple stores in a block, only the last one counts. - std::optional loc; - for (SILInstruction *I : LoadsAndStores) { - if (auto *SI = isStoreToAccess(I, accessPath)) { - loc = SI->getLoc(); - - // If a store just stores the loaded value, bail. The operand (= the load) - // will be removed later, so it cannot be used as available value. - // This corner case is surprisingly hard to handle, so we just give up. - if (isLoadWithinAccess(dyn_cast(SI->getSrc()), accessPath)) - return; - - if (!storeAddr) { - storeAddr = SI->getDest(); - ssaUpdater.initialize(storeAddr->getFunction(), - storeAddr->getType().getObjectType(), - storeAddr->getOwnershipKind()); - } else if (SI->getDest()->getType() != storeAddr->getType()) { - // This transformation assumes that the values of all stores in the loop - // must be interchangeable. It won't work if stores different types - // because of casting or payload extraction even though they have the - // same access path. - return; - } - ssaUpdater.addAvailableValue(SI->getParent(), SI->getSrc()); - } - } - assert(storeAddr && "hoistLoadsAndStores requires a store in the loop"); - auto checkBase = [&](SILValue srcAddr) { - // Clone projections until the address dominates preheader. - if (DomTree->dominates(srcAddr->getParentBlock(), preheader)) - return srcAddr; - - // return an invalid SILValue to continue cloning. - return SILValue(); - }; - SILValue initialAddr = - cloneUseDefChain(storeAddr, preheader->getTerminator(), checkBase); - // cloneUseDefChain may currently fail if a begin_borrow or mark_dependence is - // in the chain. - if (!initialAddr) - return; - - LoadOwnershipQualifier ownership = B.getFunction().hasOwnership() ? - LoadOwnershipQualifier::Trivial : - LoadOwnershipQualifier::Unqualified; - - LoadInst *initialLoad = - B.createLoad(RegularLocation::getAutoGeneratedLocation(), initialAddr, - ownership); - LLVM_DEBUG(llvm::dbgs() << "Creating preload " << *initialLoad); - ssaUpdater.addAvailableValue(preheader, initialLoad); - - // Remove all stores and replace the loads with the current value. - SILBasicBlock *currentBlock = nullptr; - SILValue currentVal; - for (SILInstruction *I : LoadsAndStores) { - SILBasicBlock *block = I->getParent(); - if (block != currentBlock) { - currentBlock = block; - currentVal = SILValue(); - } - if (auto *SI = isStoreToAccess(I, accessPath)) { - LLVM_DEBUG(llvm::dbgs() << "Deleting reloaded store " << *SI); - currentVal = SI->getSrc(); - toDelete.insert(SI); - continue; - } - auto loadWithAccess = isLoadWithinAccess(I, accessPath); - if (!loadWithAccess) { - continue; - } - // If we didn't see a store in this block yet, get the current value from - // the ssaUpdater. - if (!currentVal) - currentVal = ssaUpdater.getValueInMiddleOfBlock(block); - - LoadInst *load = loadWithAccess.li; - auto loadAddress = load->getOperand(); - SILValue projectedValue = projectLoadValue( - loadAddress, loadWithAccess.accessPath, currentVal, accessPath, load); - LLVM_DEBUG(llvm::dbgs() << "Replacing stored load " << *load << " with " - << projectedValue); - load->replaceAllUsesWith(projectedValue); - toDelete.insert(load); - } - - // Store back the value at all loop exits. - for (SILBasicBlock *exitingOrLatchBlock : exitingAndLatchBlocks) { - for (SILBasicBlock *succ : exitingOrLatchBlock->getSuccessors()) { - if (loop->contains(succ)) - continue; - - assert(succ->getSinglePredecessorBlock() - && "should have split critical edges"); - SILBuilder B(succ->begin()); - StoreOwnershipQualifier ownership = B.getFunction().hasOwnership() ? - StoreOwnershipQualifier::Trivial : - StoreOwnershipQualifier::Unqualified; - auto *SI = B.createStore( - loc.value(), ssaUpdater.getValueInMiddleOfBlock(succ), initialAddr, - ownership); - (void)SI; - LLVM_DEBUG(llvm::dbgs() << "Creating loop-exit store " << *SI); - } - } - - // In case the value is only stored but never loaded in the loop. - eliminateDeadInstruction(initialLoad); -} - -bool LoopTreeOptimization::hoistAllLoadsAndStores(SILLoop *loop) { - for (AccessPath accessPath : LoadAndStoreAddrs) { - hoistLoadsAndStores(accessPath, loop); - } - LoadsAndStores.clear(); - LoadAndStoreAddrs.clear(); - - if (toDelete.empty()) - return false; - - for (SILInstruction *I : toDelete) { - recursivelyDeleteTriviallyDeadInstructions(I, /*force*/ true); - } - toDelete.clear(); - return true; -} - -namespace { -/// Hoist loop invariant code out of innermost loops. -/// -/// Transforms are identified by type, not instance. Split this -/// Into two types: "High-level Loop Invariant Code Motion" -/// and "Loop Invariant Code Motion". -class LICM : public SILFunctionTransform { - -public: - LICM(bool RunsOnHighLevelSil) : RunsOnHighLevelSil(RunsOnHighLevelSil) {} - - /// True if LICM is done on high-level SIL, i.e. semantic calls are not - /// inlined yet. In this case some semantic calls can be hoisted. - /// We only hoist semantic calls on high-level SIL because we can be sure that - /// e.g. an Array as SILValue is really immutable (including its content). - bool RunsOnHighLevelSil; - - void run() override { - SILFunction *F = getFunction(); - - SILLoopAnalysis *LA = PM->getAnalysis(); - SILLoopInfo *LoopInfo = LA->get(F); - - if (LoopInfo->empty()) { - LLVM_DEBUG(llvm::dbgs() << "No loops in " << F->getName() << "\n"); - return; - } - - DominanceAnalysis *DA = PM->getAnalysis(); - PostDominanceAnalysis *PDA = PM->getAnalysis(); - AliasAnalysis *AA = PM->getAnalysis(F); - BasicCalleeAnalysis *BCA = PM->getAnalysis(); - AccessStorageAnalysis *ASA = getAnalysis(); - DominanceInfo *DomTree = nullptr; - - LLVM_DEBUG(llvm::dbgs() << "Processing loops in " << F->getName() << "\n"); - bool Changed = false; - - for (auto *TopLevelLoop : *LoopInfo) { - if (!DomTree) DomTree = DA->get(F); - LoopTreeOptimization Opt(TopLevelLoop, LoopInfo, AA, BCA, DomTree, PDA, - ASA, RunsOnHighLevelSil); - Changed |= Opt.optimize(); - } - - if (Changed) { - LA->lockInvalidation(); - DA->lockInvalidation(); - PM->invalidateAnalysis(F, SILAnalysis::InvalidationKind::FunctionBody); - LA->unlockInvalidation(); - DA->unlockInvalidation(); - } - } -}; -} // end anonymous namespace - -SILTransform *swift::createLICM() { - return new LICM(false); -} - -SILTransform *swift::createHighLevelLICM() { - return new LICM(true); -} diff --git a/lib/SILOptimizer/PassManager/PassPipeline.cpp b/lib/SILOptimizer/PassManager/PassPipeline.cpp index 8448ad1e217a8..7782243d14198 100644 --- a/lib/SILOptimizer/PassManager/PassPipeline.cpp +++ b/lib/SILOptimizer/PassManager/PassPipeline.cpp @@ -392,7 +392,7 @@ void addHighLevelLoopOptPasses(SILPassPipelinePlan &P) { // before CanonicalOSSA re-hoists destroys. P.addAccessEnforcementReleaseSinking(); P.addAccessEnforcementOpts(); - P.addHighLevelLICM(); + P.addLoopInvariantCodeMotion(); // Simplify CFG after LICM that creates new exit blocks P.addSimplifyCFG(); // LICM might have added new merging potential by hoisting @@ -481,7 +481,7 @@ void addFunctionPasses(SILPassPipelinePlan &P, // late as possible before inlining because it must run between runs of the // inliner when the pipeline restarts. if (OpLevel == OptimizationLevelKind::MidLevel) { - P.addHighLevelLICM(); + P.addLoopInvariantCodeMotion(); P.addArrayCountPropagation(); P.addBoundsCheckOpts(); P.addDCE(); @@ -739,10 +739,10 @@ static void addMidLevelFunctionPipeline(SILPassPipelinePlan &P) { // A LICM pass at mid-level is mainly needed to hoist addressors of globals. // It needs to be before global_init functions are inlined. - P.addLICM(); + P.addLoopInvariantCodeMotion(); // Run loop unrolling after inlining and constant propagation, because loop // trip counts may have became constant. - P.addLICM(); + P.addLoopInvariantCodeMotion(); P.addLoopUnroll(); } @@ -848,7 +848,7 @@ static void addLateLoopOptPassPipeline(SILPassPipelinePlan &P) { // It will also set the no_nested_conflict for dynamic accesses P.addAccessEnforcementReleaseSinking(); P.addAccessEnforcementOpts(); - P.addLICM(); + P.addLoopInvariantCodeMotion(); P.addCOWOpts(); // Simplify CFG after LICM that creates new exit blocks P.addSimplifyCFG(); @@ -892,7 +892,7 @@ static void addLastChanceOptPassPipeline(SILPassPipelinePlan &P) { P.addAccessEnforcementDom(); // addAccessEnforcementDom might provide potential for LICM: // A loop might have only one dynamic access now, i.e. hoistable - P.addLICM(); + P.addLoopInvariantCodeMotion(); // Verify AccessStorage once again after optimizing and lowering OSSA. #ifndef NDEBUG diff --git a/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp b/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp index 2257f526f0b9c..b96f862b23b63 100644 --- a/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp +++ b/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp @@ -686,7 +686,7 @@ recursivelyCollectInteriorUses(ValueBase *DefInst, continue; } ArraySemanticsCall AS(svi); - if (AS.getKind() == swift::ArrayCallKind::kArrayFinalizeIntrinsic) { + if (AS.getKind() == ArrayCallKind::kArrayFinalizeIntrinsic) { if (!recursivelyCollectInteriorUses(svi, AddressNode, IsInteriorAddress)) return false; continue; diff --git a/lib/SILOptimizer/Utils/OptimizerBridging.cpp b/lib/SILOptimizer/Utils/OptimizerBridging.cpp index a41d498448100..181eb1c13a70a 100644 --- a/lib/SILOptimizer/Utils/OptimizerBridging.cpp +++ b/lib/SILOptimizer/Utils/OptimizerBridging.cpp @@ -272,32 +272,31 @@ BridgedOwnedString BridgedPassContext::mangleWithDeadArgs(BridgedArrayRef bridge } BridgedOwnedString BridgedPassContext::mangleWithClosureArgs( - BridgedValueArray bridgedClosureArgs, - BridgedArrayRef bridgedClosureArgIndices, - BridgedFunction applySiteCallee + BridgedArrayRef bridgedClosureArgs, BridgedFunction applySiteCallee ) const { + + struct ClosureArgElement { + SwiftInt argIdx; + BridgeValueExistential argValue; + }; + auto pass = Demangle::SpecializationPass::ClosureSpecializer; auto serializedKind = applySiteCallee.getFunction()->getSerializedKind(); Mangle::FunctionSignatureSpecializationMangler mangler(applySiteCallee.getFunction()->getASTContext(), pass, serializedKind, applySiteCallee.getFunction()); - llvm::SmallVector closureArgsStorage; - auto closureArgs = bridgedClosureArgs.getValues(closureArgsStorage); - auto closureArgIndices = bridgedClosureArgIndices.unbridged(); - - assert(closureArgs.size() == closureArgIndices.size() && - "Number of closures arguments and number of closure indices do not match!"); + auto closureArgs = bridgedClosureArgs.unbridged(); - for (size_t i = 0; i < closureArgs.size(); i++) { - auto closureArg = closureArgs[i]; - auto closureArgIndex = closureArgIndices[i]; + for (ClosureArgElement argElmt : closureArgs) { + auto closureArg = argElmt.argValue.value.getSILValue(); + auto closureArgIndex = argElmt.argIdx; if (auto *PAI = dyn_cast(closureArg)) { mangler.setArgumentClosureProp(closureArgIndex, const_cast(PAI)); } else { auto *TTTFI = cast(closureArg); - mangler.setArgumentClosureProp(closureArgIndex, + mangler.setArgumentClosureProp(closureArgIndex, const_cast(TTTFI)); } } @@ -459,43 +458,12 @@ BridgedCalleeAnalysis::CalleeList BridgedCalleeAnalysis::getDestructors(BridgedT return ca->getDestructors(type.unbridged(), isExactType); } -namespace swift { - class SpecializationCloner: public SILClonerWithScopes { - friend class SILInstructionVisitor; - friend class SILCloner; - public: - using SuperTy = SILClonerWithScopes; - SpecializationCloner(SILFunction &emptySpecializedFunction): SuperTy(emptySpecializedFunction) {} - }; -} // namespace swift - -BridgedSpecializationCloner::BridgedSpecializationCloner(BridgedFunction emptySpecializedFunction): - cloner(new SpecializationCloner(*emptySpecializedFunction.getFunction())) {} - -BridgedFunction BridgedSpecializationCloner::getCloned() const { - return { &cloner->getBuilder().getFunction() }; -} - -BridgedBasicBlock BridgedSpecializationCloner::getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const { - return { cloner->getOpBasicBlock(originalBasicBlock.unbridged()) }; -} - -void BridgedSpecializationCloner::cloneFunctionBody(BridgedFunction originalFunction, BridgedBasicBlock clonedEntryBlock, BridgedValueArray clonedEntryBlockArgs) const { - llvm::SmallVector clonedEntryBlockArgsStorage; - auto clonedEntryBlockArgsArrayRef = clonedEntryBlockArgs.getValues(clonedEntryBlockArgsStorage); - cloner->cloneFunctionBody(originalFunction.getFunction(), clonedEntryBlock.unbridged(), clonedEntryBlockArgsArrayRef); -} - -void BridgedSpecializationCloner::cloneFunctionBody(BridgedFunction originalFunction) const { - cloner->cloneFunction(originalFunction.getFunction()); -} - void BridgedBuilder::destroyCapturedArgs(BridgedInstruction partialApply) const { if (auto *pai = llvm::dyn_cast(partialApply.unbridged()); pai->isOnStack()) { auto b = unbridged(); - return swift::insertDestroyOfCapturedArguments(pai, b); + return swift::insertDestroyOfCapturedArguments(pai, b); } else { - assert(false && "`destroyCapturedArgs` must only be called on a `partial_apply` on stack!"); + assert(false && "`destroyCapturedArgs` must only be called on a `partial_apply` on stack!"); } } diff --git a/test/SILOptimizer/high_level_licm.sil b/test/SILOptimizer/high_level_licm.sil index 5c13725ca648f..9eead6190eb67 100644 --- a/test/SILOptimizer/high_level_licm.sil +++ b/test/SILOptimizer/high_level_licm.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -enable-sil-verify-all %s -high-level-licm | %FileCheck %s +// RUN: %target-sil-opt -enable-sil-verify-all %s -loop-invariant-code-motion | %FileCheck %s sil_stage canonical diff --git a/test/SILOptimizer/inlinearray_bounds_check_tests.swift b/test/SILOptimizer/inlinearray_bounds_check_tests.swift index 49e75f2340379..1e23e65bf4485 100644 --- a/test/SILOptimizer/inlinearray_bounds_check_tests.swift +++ b/test/SILOptimizer/inlinearray_bounds_check_tests.swift @@ -1,8 +1,8 @@ // RUN: %target-swift-frontend %s -emit-sil -O \ -// RUN: -disable-availability-checking | %FileCheck %s --check-prefix=CHECK-SIL +// RUN: -disable-availability-checking | %FileCheck %s --check-prefix=CHECK-SIL // RUN: %target-swift-frontend %s -emit-ir -O \ -// RUN: -disable-availability-checking | %FileCheck %s --check-prefix=CHECK-IR +// RUN: -disable-availability-checking | %FileCheck %s --check-prefix=CHECK-IR // REQUIRES: swift_in_compiler // REQUIRES: swift_stdlib_no_asserts, optimized_stdlib @@ -131,7 +131,7 @@ public func inlinearray_sum_iterate_to_deducible_count2_wo_trap(_ v: // Bounds check should be eliminated -// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A42_sum_iterate_to_deducible_count2_with_trapySis11InlineArrayVyxSiG_SitSiRVzlF : +// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A42_sum_iterate_to_deducible_count2_with_trapySis11InlineArrayVyxSiG_SitSiRVzlF : // CHECK-SIL: bb3 // CHECK-SIL-NOT: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_br @@ -147,7 +147,7 @@ public func inlinearray_sum_iterate_to_deducible_count2_with_trap(_ // Bounds check should be eliminated -// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A29_iterate_over_indices_wo_trapySis11InlineArrayVyxSiGSiRVzlF : +// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A29_iterate_over_indices_wo_trapySis11InlineArrayVyxSiGSiRVzlF : // CHECK-SIL: bb3 // CHECK-SIL-NOT: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_br @@ -166,7 +166,7 @@ public func inlinearray_iterate_over_indices_wo_trap(_ v: InlineArra // Bounds check should be eliminated // Induction variable optimization eliminates the bounds check in SIL -// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A31_iterate_over_indices_with_trapySis11InlineArrayVyxSiGSiRVzlF : +// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A31_iterate_over_indices_with_trapySis11InlineArrayVyxSiGSiRVzlF : // CHECK-SIL: bb3 // CHECK-SIL-NOT: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_br @@ -181,7 +181,7 @@ public func inlinearray_iterate_over_indices_with_trap(_ v: InlineAr // Eliminate duplicate bounds check -// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A17_element_equalityySbs11InlineArrayVyxSiG_SitSiRVzlF : +// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A17_element_equalityySbs11InlineArrayVyxSiG_SitSiRVzlF : // CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL-NOT: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL-LABEL: } // end sil function '$s30inlinearray_bounds_check_tests0A17_element_equalityySbs11InlineArrayVyxSiG_SitSiRVzlF' @@ -201,7 +201,7 @@ public func inlinearray_element_sum(_ v: InlineArray, _ i: I // Bounds check should be eliminated -// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A7_searchySiSgs11InlineArrayVyq_xG_xtSiRV_SQRzr0_lF : +// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A7_searchySiSgs11InlineArrayVyq_xG_xtSiRV_SQRzr0_lF : // CHECK-SIL: bb3: // CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" @@ -218,7 +218,7 @@ public func inlinearray_search(_ v: InlineArray // Bounds check should be eliminated -// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A11_search_splySiSgs11InlineArrayVyxSiG_SitSiRVzlF : +// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A11_search_splySiSgs11InlineArrayVyxSiG_SitSiRVzlF : // CHECK-SIL: bb3: // CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" @@ -235,7 +235,7 @@ public func inlinearray_search_spl(_ v: InlineArray, _ elem: // Bounds check should be eliminated -// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A18_binary_search_splySiSgs11InlineArrayVyxSiG_SitSiRVzlF : +// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A18_binary_search_splySiSgs11InlineArrayVyxSiG_SitSiRVzlF : // CHECK-SIL: bb2 // CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_br @@ -262,7 +262,7 @@ public func inlinearray_binary_search_spl(_ v: InlineArray, // This prevents LoopRotate which prevent bounds checks opts since it depends on induction variable analysis which doesn't work on unrotated loops. // CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A35_sum_iterate_to_count_with_trap_splySis11InlineArrayVy$63_SiGF : // CHECK-SIL: bb2 -// CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" +// CHECK-NOT-SIL: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_br // CHECK-SIL-LABEL: } // end sil function '$s30inlinearray_bounds_check_tests0A35_sum_iterate_to_count_with_trap_splySis11InlineArrayVy$63_SiGF' public func inlinearray_sum_iterate_to_count_with_trap_spl(_ v: InlineArray<64, Int>) -> Int { @@ -274,7 +274,7 @@ public func inlinearray_sum_iterate_to_count_with_trap_spl(_ v: InlineArray<64, } // InlineArray is copied into a temporary within the loop in the "specialized" version -// This prevents LoopRotate which prevent bounds checks opts since it depends on induction variable analysis which doesn't work on unrotated loops. +// This prevents LoopRotate which prevent bounds checks opts since it depends on induction variable analysis which doesn't work on unrotated loops. // CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A37_sum_iterate_to_unknown_with_trap_splySis11InlineArrayVy$63_SiG_SitF : // CHECK-SIL: bb2 // CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" @@ -290,7 +290,7 @@ public func inlinearray_sum_iterate_to_unknown_with_trap_spl(_ v: InlineArray<64 // Current codegen for this in SIL is very poor // First a temp is created and the elements are stored to it, then they get loaded from the temp to be stored in the let which is then loaded again to get the sum -// However, LLVM can constant fold everything +// However, LLVM can constant fold everything public func local_inlinearray_sum_iterate_to_count_with_trap_spl() -> Int { var sum = 0 let v : InlineArray = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16] @@ -302,7 +302,7 @@ public func local_inlinearray_sum_iterate_to_count_with_trap_spl() -> Int { // Current codegen for this in SIL is very poor // First a temp is created and the elements are stored to it, then they get loaded from the temp to be stored in the let which is then loaded again to get the sum -// LLVM cannot constant fold, it memsets, memcopies and then loops over to sum +// LLVM cannot constant fold, it memsets, memcopies and then loops over to sum public func local_inlinearray_repeating_init_sum_iterate_to_count_trap_spl() -> Int { var sum = 0 let v = InlineArray<64, Int>(repeating: 64) @@ -324,9 +324,9 @@ public func inlinearray_inc_by_one(_ v: inout InlineArray, _ } } -// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A15_inc_by_one_splyys11InlineArrayVy$63_SiGz_SitF : +// CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A15_inc_by_one_splyys11InlineArrayVy$63_SiGz_SitF : // CHECK-SIL: bb2 -// CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" +// CHECK-NOT-SIL: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_br // CHECK-SIL-LABEL: } // end sil function '$s30inlinearray_bounds_check_tests0A15_inc_by_one_splyys11InlineArrayVy$63_SiGz_SitF' public func inlinearray_inc_by_one_spl(_ v: inout InlineArray<64, Int>, _ n: Int) { diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index 0dbf019064002..73c581511fa46 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=none -enable-sil-verify-all %s -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=none -enable-sil-verify-all %s -loop-invariant-code-motion | %FileCheck %s // REQUIRES: swift_in_compiler @@ -182,13 +182,14 @@ bb4: sil [_semantics "array.get_count"] @getCount : $@convention(method) (@guaranteed Array) -> Int sil @user : $@convention(thin) (Int) -> () -// CHECK-LABEL: sil @dont_hoist_get_count_on_low_level_sil -// CHECK: {{^}}bb1: +// CHECK-LABEL: sil @hoist_get_count_on_low_level_sil +// CHECK: {{^}}bb0(%0 : $Array): // CHECK: apply +// CHECK: {{^}}bb1: // CHECK: apply // CHECK: {{^}}bb2: // CHECK: return -sil @dont_hoist_get_count_on_low_level_sil : $@convention(thin) (@guaranteed Array) -> () { +sil @hoist_get_count_on_low_level_sil : $@convention(thin) (@guaranteed Array) -> () { bb0(%0 : $Array): br bb1 @@ -943,12 +944,9 @@ bb5: return %99 : $() } -// Test load splitting with a loop-invariant stored value. The loop +// Test load splitting with a loop-invariant. The loop // will be empty after combined load/store hoisting/sinking. // -// TODO: sink a struct_extract (or other non-side-effect instructions) -// with no uses in the loop. -// // CHECK-LABEL: sil shared @testLoadSplit : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { // CHECK: [[PRELOAD:%.*]] = load %{{.*}} : $*Int64 // CHECK: [[STOREDVAL:%.*]] = struct_extract %0 : $Int64, #Int64._value @@ -984,12 +982,12 @@ bb2: bb3: %result = tuple (%val1 : $Index, %val2 : $Int64, %val3 : $Builtin.Int64) return %result : $(Index, Int64, Builtin.Int64) -} +} // end sil function 'testLoadSplit' // Test load splitting with a loop-varying stored value. // CHECK-LABEL: sil shared @testLoadSplitPhi : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { // CHECK: [[PRELOAD:%.*]] = load %{{.*}} : $*Int64 -// CHECK: br bb1(%4 : $Int64) +// CHECK: br bb1(%{{.*}} : $Int64) // CHECK: bb1([[PHI:%.*]] : $Int64): // CHECK-NEXT: [[OUTERVAL:%.*]] = struct $Index ([[PHI]] : $Int64) // CHECK-NEXT: [[EXTRACT:%.*]] = struct_extract [[PHI]] : $Int64, #Int64._value @@ -1030,7 +1028,7 @@ bb2: bb3: %result = tuple (%outerVal : $Index, %middleVal : $Int64, %innerVal : $Builtin.Int64) return %result : $(Index, Int64, Builtin.Int64) -} +} // end sil function 'testLoadSplitPhi' struct State { @_hasStorage var valueSet: (Int64, Int64, Int64) { get set } @@ -1045,11 +1043,12 @@ struct State { // CHECK: bb0(%0 : $Builtin.RawPointer): // CHECK: [[HOISTADR:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 0 // ...Preload stored element #1 +// CHECK: tuple_element_addr // CHECK: [[PRELOADADR:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 1 // CHECK: [[PRELOAD:%.*]] = load [[PRELOADADR]] : $*Int64 // ...Split element 0 -// CHECK: [[SPLIT0:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 0 -// CHECK: [[ELT0:%.*]] = load [[SPLIT0]] : $*Int64 +// CHECK: [[ELT0:%.*]] = load [[HOISTADR]] : $*Int64 +// CHECK: [[HOISTVAL:%.*]] = struct_extract [[ELT0]] : $Int64, #Int64._value // ...Split element 2 // CHECK: [[SPLIT2:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 2 // CHECK: [[ELT2:%.*]] = load [[SPLIT2]] : $*Int64 @@ -1057,8 +1056,6 @@ struct State { // CHECK: [[SINGLEADR:%.*]] = struct_element_addr %{{.*}} : $*State, #State.singleValue // CHECK: [[SINGLEVAL:%.*]] = load [[SINGLEADR]] : $*Int64 // ...Hoisted element 0 -// CHECK: [[HOISTLOAD:%.*]] = load [[HOISTADR]] : $*Int64 -// CHECK: [[HOISTVAL:%.*]] = struct_extract [[HOISTLOAD]] : $Int64, #Int64._value // CHECK: br bb1([[PRELOAD]] : $Int64) // ...Loop // CHECK: bb1([[PHI:%.*]] : $Int64): @@ -1166,7 +1163,6 @@ bb3: // CHECK: [[ELT_1a:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, (Int64, Int64)), 1 // CHECK: [[ELT_1_0:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64), 0 // CHECK: [[V_1_0:%.*]] = load [[ELT_1_0]] : $*Int64 -// CHECK: [[ELT_1b:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, (Int64, Int64)), 1 // CHECK: [[ELT_1_1:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64), 1 // CHECK: [[V_1_1:%.*]] = load [[ELT_1_1]] : $*Int64 // CHECK: br bb1([[V_0:%.*]] : $Int64, [[V_1_0]] : $Int64) @@ -1241,25 +1237,24 @@ bb3: return %val1 : $(Int64, (Int64, Int64)) } -// Two wide loads. The first can be successfully split and the second -// half hoisted. The second cannot be split because of a pointer -// cast. Make sure two remaining loads and the store are still in the loop. +// Split two wide loads. // // CHECK-LABEL: sil hidden @testSplitNonStandardProjection : $@convention(method) (Int64, Builtin.RawPointer) -> ((Int64, (Int64, Int64)), (Int64, Int64)) { // CHECK: bb0(%0 : $Int64, %1 : $Builtin.RawPointer): // -// The first load was split, so one half is hoisted. -// CHECK: [[V1:%.*]] = load %{{.*}} : $*Int64 -// CHECK: br bb1 -// CHECK: bb1: -// CHECK: [[V0:%.*]] = load %{{.*}} : $*Int64 -// CHECK: [[INNER:%.*]] = tuple ([[V0]] : $Int64, [[V1]] : $Int64) -// CHECK: store %0 to %{{.*}} : $*Int64 -// CHECK: [[OUTER:%.*]] = load %{{.*}} : $*(Int64, (Int64, Int64)) +// CHECK: [[ADDR1:%.*]] = tuple_element_addr %{{.*}}, 0 +// CHECK: [[V1:%.*]] = load [[ADDR1]] : $*Int64 +// CHECK: [[ADDR0:%.*]] = tuple_element_addr %{{.*}}, 1 +// CHECK: [[V0:%.*]] = load [[ADDR0]] : $*Int64 +// CHECK: [[OUTER:%.*]] = tuple (%{{.*}} : $Int64, %{{.*}} : $(Int64, Int64)) +// CHECK: br bb1([[V1]] : $Int64) +// CHECK: bb1([[PHI:%.*]] : $Int64): +// CHECK: [[INNER:%.*]] = tuple ([[PHI]] : $Int64, [[V0]] : $Int64) // CHECK: cond_br undef, bb2, bb3 // CHECK: bb2: -// CHECK: br bb1 +// CHECK: br bb1(%0 : $Int64) // CHECK: bb3: +// CHECK: store %0 to [[ADDR1]] : $*Int64 // CHECK: [[RESULT:%.*]] = tuple ([[OUTER]] : $(Int64, (Int64, Int64)), [[INNER]] : $(Int64, Int64)) // CHECK: return [[RESULT]] : $((Int64, (Int64, Int64)), (Int64, Int64)) // CHECK-LABEL: } // end sil function 'testSplitNonStandardProjection' @@ -1291,11 +1286,11 @@ bb3: // CHECK: bb0(%0 : $Int64, %1 : $Int64, %2 : $Builtin.RawPointer): // CHECK: [[ELT_1:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 1 // CHECK: [[V1:%.*]] = load %4 : $*(Int64, Int64) -// CHECK: [[ELT_0:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 0 -// CHECK: [[V0:%.*]] = load %6 : $*Int64 // CHECK: [[ARG0:%.*]] = tuple (%0 : $Int64, %0 : $Int64) -// CHECK: [[ARG0_0:%.*]] = tuple_extract %8 : $(Int64, Int64), 0 // CHECK: [[ARG1:%.*]] = tuple (%1 : $Int64, %1 : $Int64) +// CHECK: [[ELT_0:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 0 +// CHECK: [[V0:%.*]] = load %9 : $*Int64 +// CHECK: [[ARG0_0:%.*]] = tuple_extract [[ARG0]] : $(Int64, Int64), 0 // CHECK: br bb1([[V1]] : $(Int64, Int64)) // CHECK: bb1([[PHI:%.*]] : $(Int64, Int64)): // CHECK: [[LOOPVAL:%.*]] = tuple ([[V0]] : $Int64, [[PHI]] : $(Int64, Int64)) @@ -1343,8 +1338,10 @@ class C {} // This won't be hoisted because we can't find a base to check if it is invariant // CHECK-LABEL: sil @testLoopInvariantStoreNoBase1 : +// CHECK: bb3(%11 : $Builtin.RawPointer): +// CHECK-NOT: load // CHECK: bb6: -// CHECK-NOT: store +// CHECK: store // CHECK-LABEL: } // end sil function 'testLoopInvariantStoreNoBase1' sil @testLoopInvariantStoreNoBase1 : $@convention(thin) (Builtin.BridgeObject, Double) -> () { bb0(%0 : $Builtin.BridgeObject, %1 : $Double): @@ -1380,8 +1377,10 @@ bb6: // This won't be hoisted because we can't find a base to check if it is invariant // CHECK-LABEL: sil @testLoopInvariantStoreNoBase2 : +// CHECK: bb3(%11 : $Builtin.RawPointer): +// CHECK-NOT: load // CHECK: bb6: -// CHECK-NOT: store +// CHECK: store // CHECK-LABEL: } // end sil function 'testLoopInvariantStoreNoBase2' sil @testLoopInvariantStoreNoBase2 : $@convention(thin) (Builtin.BridgeObject, Double) -> () { bb0(%0 : $Builtin.BridgeObject, %1 : $Double): @@ -1561,7 +1560,7 @@ bb3: // CHECK-LABEL: sil [ossa] @hoist_trivial_load : // CHECK: load [trivial] -// CHECK-NEXT: br bb1 +// CHECK-NEXT: br bb1 // CHECK: } // end sil function 'hoist_trivial_load' sil [ossa] @hoist_trivial_load : $@convention(thin) (@inout Int) -> () { bb0(%0 : $*Int): @@ -1657,7 +1656,7 @@ bb3: // CHECK-LABEL: } // end sil function 'store_of_optional_none' sil [ossa] @store_of_optional_none : $@convention(thin) () -> () { bb0: - %0 = enum $Optional, #Optional.none!enumelt + %0 = enum $Optional, #Optional.none!enumelt %1 = alloc_stack $Optional store %0 to [init] %1 br bb1 @@ -1733,5 +1732,3 @@ bb3: %13 = tuple () return %13 } - - diff --git a/test/SILOptimizer/licm_apply.sil b/test/SILOptimizer/licm_apply.sil index df5a5381a2731..77fe0bf6b5a4f 100644 --- a/test/SILOptimizer/licm_apply.sil +++ b/test/SILOptimizer/licm_apply.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -compute-side-effects -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -compute-side-effects -loop-invariant-code-motion | %FileCheck %s // REQUIRES: swift_in_compiler diff --git a/test/SILOptimizer/licm_exclusivity.sil b/test/SILOptimizer/licm_exclusivity.sil index bd7049cc7bc06..fcfe9770af4be 100644 --- a/test/SILOptimizer/licm_exclusivity.sil +++ b/test/SILOptimizer/licm_exclusivity.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=checked -enable-sil-verify-all %s -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=checked -enable-sil-verify-all %s -loop-invariant-code-motion | %FileCheck %s sil_stage canonical diff --git a/test/SILOptimizer/licm_exclusivity.swift b/test/SILOptimizer/licm_exclusivity.swift index 458a57c42e0d8..0b9f56e6bd889 100644 --- a/test/SILOptimizer/licm_exclusivity.swift +++ b/test/SILOptimizer/licm_exclusivity.swift @@ -1,17 +1,9 @@ -// RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -Xllvm -debug-only=sil-licm -primary-file %s 2>&1 | %FileCheck %s --check-prefix=TESTLICM -// RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -Xllvm -debug-only=sil-licm -primary-file %s 2>&1 | %FileCheck %s --check-prefix=TESTLICM2 // RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -primary-file %s | %FileCheck %s --check-prefix=TESTSIL -// RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -Xllvm -debug-only=sil-licm -whole-module-optimization %s 2>&1 | %FileCheck %s --check-prefix=TESTLICMWMO // RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -whole-module-optimization %s | %FileCheck %s --check-prefix=TESTSILWMO // REQUIRES: optimized_stdlib,asserts,swift_stdlib_no_asserts // REQUIRES: PTRSIZE=64 -// TESTLICM-LABEL: Processing loops in {{.*}}run_ReversedArray{{.*}} -// TESTLICM: Hoist and Sink pairs attempt -// TESTLICM: Hoisted -// TESTLICM: Successfully hoisted and sank pair - // TESTSIL-LABEL: sil hidden @$s16licm_exclusivity17run_ReversedArrayyySiF : $@convention(thin) (Int) -> () { // TESTSIL: bb // TESTSIL: begin_access [modify] [dynamic] [no_nested_conflict] @@ -35,10 +27,6 @@ func run_ReversedArray(_ N: Int) { } } -// TESTLICM2-LABEL: Processing loops in {{.*}}count_unicodeScalars{{.*}} -// TESTLICM2: Hoist and Sink pairs attempt -// TESTLICM2: Hoisted - // TESTSIL-LABEL: sil @$s16licm_exclusivity20count_unicodeScalarsyySS17UnicodeScalarViewVF : $@convention(thin) (@guaranteed String.UnicodeScalarView) -> () { // TESTSIL: bb0(%0 : $String.UnicodeScalarView) // TESTSIL: bb5: @@ -66,12 +54,6 @@ public class ClassWithArrs { B = [Int](repeating: 0, count: N) } -// TESTLICMWMO-LABEL: Processing loops in {{.*}}ClassWithArrsC7readArr{{.*}} -// TESTLICMWMO: Hoist and Sink pairs attempt -// TESTLICMWMO: Hoisted -// TESTLICMWMO: Successfully hoisted and sank pair -// TESTLICMWMO: Hoisted -// TESTLICMWMO: Successfully hoisted and sank pair // TESTSILWMO-LABEL: sil {{.*}}@$s16licm_exclusivity13ClassWithArrsC7readArryyF : $@convention(method) (@guaranteed ClassWithArrs) -> () { // TESTSILWMO: [[R1:%.*]] = ref_element_addr %0 : $ClassWithArrs, #ClassWithArrs.A // TESTSILWMO: [[R2:%.*]] = ref_element_addr %0 : $ClassWithArrs, #ClassWithArrs.B diff --git a/test/SILOptimizer/licm_multiend.sil b/test/SILOptimizer/licm_multiend.sil index 32d83b3d14ad6..5a22419f66b5f 100644 --- a/test/SILOptimizer/licm_multiend.sil +++ b/test/SILOptimizer/licm_multiend.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -loop-invariant-code-motion | %FileCheck %s // REQUIRES: PTRSIZE=64 // REQUIRES: OS=macosx diff --git a/test/SILOptimizer/licm_unreferenceablestorage.sil b/test/SILOptimizer/licm_unreferenceablestorage.sil index d4d4686344fb7..6a93057fb51f5 100644 --- a/test/SILOptimizer/licm_unreferenceablestorage.sil +++ b/test/SILOptimizer/licm_unreferenceablestorage.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -loop-invariant-code-motion | %FileCheck %s // REQUIRES: objc_interop