diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt index bedf3296a61df..4f95a146351ba 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/CMakeLists.txt @@ -9,6 +9,7 @@ swift_compiler_sources(Optimizer AliasAnalysis.swift CalleeAnalysis.swift + LoopTree.swift DeadEndBlocksAnalysis.swift DominatorTree.swift PostDominatorTree.swift) diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift index c7c106d879c08..132434c5eb028 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/DominatorTree.swift @@ -15,6 +15,31 @@ import OptimizerBridging struct DominatorTree { let bridged: BridgedDomTree + + func getChildren(of block: BasicBlock) -> DomChildren { + return DomChildren(bridgedDomTree: bridged, bb: block) + } +} + +struct DomChildren: BridgedRandomAccessCollection { + private let bridgedDomTree: BridgedDomTree + private let block: BasicBlock + + let count: Int + + var startIndex: Int { return 0 } + var endIndex: Int { return count } + + init(bridgedDomTree: BridgedDomTree, bb: BasicBlock) { + self.bridgedDomTree = bridgedDomTree + self.block = bb + self.count = bridgedDomTree.getNumberOfChildren(bb.bridged) + } + + subscript(_ index: Int) -> BasicBlock { + assert(index >= startIndex && index < endIndex) + return bridgedDomTree.getChildAt(block.bridged, index).block + } } extension BasicBlock { diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift new file mode 100644 index 0000000000000..aab4605926c3b --- /dev/null +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/LoopTree.swift @@ -0,0 +1,213 @@ +//===--- LoopTree.swift ---------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SIL +import OptimizerBridging + +/// Describes top level loops. +struct LoopTree { + fileprivate let bridged: BridgedLoopTree + + let loops: TopLevelLoopArray + + init(bridged: BridgedLoopTree, context: FunctionPassContext) { + self.bridged = bridged + self.loops = TopLevelLoopArray(bridged) + } +} + +/// Describes a loop with its children. +struct Loop { + private let bridged: BridgedLoop + + let innerLoops: LoopArray + let loopBlocks: LoopBlocks + + var exitingAndLatchBlocks: some Sequence { + return header.predecessors.lazy + .filter { predecessor in + contains(block: predecessor) && !isLoopExiting(loopBlock: predecessor) + } + exitingBlocks + } + + /// Exit blocks of the loop. + /// + /// - Note: Some exit blocks will be duplicated if the loop has critical edges. + var exitBlocks: some Sequence { + return loopBlocks.lazy + .flatMap(\.successors) + .filter { !contains(block: $0) } + } + + var exitingBlocks: some Sequence { + return loopBlocks.lazy + .filter { isLoopExiting(loopBlock: $0) } + } + + init(bridged: BridgedLoop) { + self.bridged = bridged + self.innerLoops = LoopArray(bridged) + self.loopBlocks = LoopBlocks(bridged) + } + + var preheader: BasicBlock? { + bridged.getPreheader().block + } + + var header: BasicBlock { + bridged.getHeader().block + } + + /// Returns `true` if the loop has exactly one exit block. + var hasSingleExitBlock: Bool { + return exitBlocks.singleElement != nil + } + + var hasNoExitBlocks: Bool { + return exitBlocks.isEmpty + } + + private func isLoopExiting(loopBlock: BasicBlock) -> Bool { + return loopBlock.successors.contains { !contains(block: $0) } + } + + func getBlocksThatDominateAllExitingAndLatchBlocks(_ context: FunctionPassContext) -> [BasicBlock] { + var result: [BasicBlock] = [] + var cachedExitingAndLatchBlocks = Stack(context) + var workList = BasicBlockWorklist(context) + defer { + cachedExitingAndLatchBlocks.deinitialize() + workList.deinitialize() + } + + cachedExitingAndLatchBlocks.append(contentsOf: exitingAndLatchBlocks) + workList.pushIfNotVisited(header) + + while let block = workList.pop() { + guard cachedExitingAndLatchBlocks.allSatisfy({ exitBlock in + return block.dominates(exitBlock, context.dominatorTree) + }) else { + continue + } + + result.append(block) + + workList.pushIfNotVisited(contentsOf: context.dominatorTree.getChildren(of: block)) + } + + return result + } + + func contains(block: BasicBlock) -> Bool { + return bridged.contains(block.bridged) + } + + func splitCriticalExitingAndBackEdges(_ context: FunctionPassContext) { + for exitingOrLatchBlock in exitingAndLatchBlocks { + for (index, succesor) in exitingOrLatchBlock.successors.enumerated() where !contains(block: succesor) { + splitCriticalEdge( + from: exitingOrLatchBlock.terminator.parentBlock, + toEdgeIndex: index, + dominatorTree: context.dominatorTree, + loopTree: context.loopTree, + context + ) + } + } + } +} + +struct TopLevelLoopArray: BridgedRandomAccessCollection { + private let bridgedLoopTree: BridgedLoopTree + + let count: Int + + var startIndex: Int { return 0 } + var endIndex: Int { return count } + + init(_ bridgedLoopTree: BridgedLoopTree) { + self.bridgedLoopTree = bridgedLoopTree + self.count = bridgedLoopTree.getTopLevelLoopCount() + } + + subscript(_ index: Int) -> Loop { + assert(index >= startIndex && index < endIndex) + return Loop(bridged: bridgedLoopTree.getLoop(index)) + } +} + +struct LoopArray: BridgedRandomAccessCollection { + private let bridgedLoop: BridgedLoop + + let count: Int + + var startIndex: Int { return 0 } + var endIndex: Int { return count } + + init(_ bridgedLoop: BridgedLoop) { + self.bridgedLoop = bridgedLoop + self.count = bridgedLoop.getInnerLoopCount() + } + + subscript(_ index: Int) -> Loop { + assert(index >= startIndex && index < endIndex) + return Loop(bridged: bridgedLoop.getInnerLoop(index)) + } +} + +struct LoopBlocks: BridgedRandomAccessCollection { + private let bridgedLoop: BridgedLoop + + let count: Int + + var startIndex: Int { return 0 } + var endIndex: Int { return count } + + init(_ bridgedLoop: BridgedLoop) { + self.bridgedLoop = bridgedLoop + self.count = bridgedLoop.getBasicBlockCount() + } + + subscript(_ index: Int) -> BasicBlock { + assert(index >= startIndex && index < endIndex) + return bridgedLoop.getBasicBlock(index).block + } +} + +func splitEdge( + from block: BasicBlock, + toEdgeIndex: Int, + dominatorTree: DominatorTree, + loopTree: LoopTree, + _ context: some MutatingContext +) -> BasicBlock { + let result = loopTree.bridged.splitEdge(block.bridged, toEdgeIndex, dominatorTree.bridged).block + + context.notifyBranchesChanged() + return result +} + +/// If the specified edge is critical, the function returns inserted block. Otherwise returns `nil`. +@discardableResult +func splitCriticalEdge( + from block: BasicBlock, + toEdgeIndex: Int, + dominatorTree: DominatorTree, + loopTree: LoopTree, + _ context: some MutatingContext +) -> BasicBlock? { + guard block.isCriticalEdge(edgeIndex: toEdgeIndex) else { + return nil + } + + return splitEdge(from: block, toEdgeIndex: toEdgeIndex, dominatorTree: dominatorTree, loopTree: loopTree, context) +} diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift index 5d09c6d9a137f..0f99978d8d2b9 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/AllocBoxToStack.swift @@ -311,10 +311,9 @@ private struct FunctionSpecializations { // This can happen if a previous run of the pass already created this specialization. return } - let cloner = SpecializationCloner(emptySpecializedFunction: specializedFunc, context) - cloner.cloneFunctionBody(from: original) - context.buildSpecializedFunction(specializedFunction: specializedFunc) { (specializedFunc, specContext) in + cloneFunction(from: original, toEmpty: specializedFunc, specContext) + replaceBoxWithStackArguments(in: specializedFunc, original: original, specContext) } context.notifyNewFunction(function: specializedFunc, derivedFrom: original) diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt index 52fe55235aee5..c0d7d0fada55e 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/CMakeLists.txt @@ -25,6 +25,7 @@ swift_compiler_sources(Optimizer LifetimeDependenceDiagnostics.swift LifetimeDependenceInsertion.swift LifetimeDependenceScopeFixup.swift + LoopInvariantCodeMotion.swift ObjectOutliner.swift ObjCBridgingOptimization.swift MergeCondFails.swift diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift index fce5c35365d19..be811c64ebf6d 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ClosureSpecialization.swift @@ -247,9 +247,10 @@ private func getOrCreateSpecializedFunction( context.buildSpecializedFunction( specializedFunction: specializedFunction, buildFn: { (emptySpecializedFunction, functionPassContext) in - let closureSpecCloner = SpecializationCloner( - emptySpecializedFunction: emptySpecializedFunction, functionPassContext) + var closureSpecCloner = Cloner( + cloneToEmptyFunction: emptySpecializedFunction, functionPassContext) closureSpecCloner.cloneAndSpecializeFunctionBody(using: pullbackClosureInfo) + closureSpecCloner.deinitialize() }) return (specializedFunction, false) @@ -731,15 +732,14 @@ private func markConvertedAndReabstractedClosuresAsUsed( } } -extension SpecializationCloner { +extension Cloner where Context == FunctionPassContext { fileprivate func cloneAndSpecializeFunctionBody(using pullbackClosureInfo: PullbackClosureInfo) { self.cloneEntryBlockArgsWithoutOrigClosures(usingOrigCalleeAt: pullbackClosureInfo) let (allSpecializedEntryBlockArgs, closureArgIndexToAllClonedReleasableClosures) = cloneAllClosures(at: pullbackClosureInfo) - self.cloneFunctionBody( - from: pullbackClosureInfo.pullbackFn, entryBlockArguments: allSpecializedEntryBlockArgs) + self.cloneFunctionBody(from: pullbackClosureInfo.pullbackFn, entryBlockArguments: allSpecializedEntryBlockArgs) self.insertCleanupCodeForClonedReleasableClosures( from: pullbackClosureInfo, @@ -750,8 +750,8 @@ extension SpecializationCloner { usingOrigCalleeAt pullbackClosureInfo: PullbackClosureInfo ) { let originalEntryBlock = pullbackClosureInfo.pullbackFn.entryBlock - let clonedFunction = self.cloned - let clonedEntryBlock = self.entryBlock + let clonedFunction = self.targetFunction + let clonedEntryBlock = self.getOrCreateEntryBlock() originalEntryBlock.arguments .enumerated() @@ -782,7 +782,8 @@ extension SpecializationCloner { ) { func entryBlockArgsWithOrigClosuresSkipped() -> [Value?] { - var clonedNonClosureEntryBlockArgs = self.entryBlock.arguments.makeIterator() + let clonedEntryBlock = self.getOrCreateEntryBlock() + var clonedNonClosureEntryBlockArgs = clonedEntryBlock.arguments.makeIterator() return pullbackClosureInfo.pullbackFn .entryBlock @@ -823,8 +824,8 @@ extension SpecializationCloner { { let (origToClonedValueMap, capturedArgRange) = self.addEntryBlockArgs( forValuesCapturedBy: closureArgDesc) - let clonedFunction = self.cloned - let clonedEntryBlock = self.entryBlock + let clonedFunction = self.targetFunction + let clonedEntryBlock = self.getOrCreateEntryBlock() let clonedClosureArgs = Array(clonedEntryBlock.arguments[capturedArgRange]) let builder = @@ -853,8 +854,8 @@ extension SpecializationCloner { -> (origToClonedValueMap: [HashableValue: Value], capturedArgRange: Range) { var origToClonedValueMap: [HashableValue: Value] = [:] - let clonedFunction = self.cloned - let clonedEntryBlock = self.entryBlock + let clonedFunction = self.targetFunction + let clonedEntryBlock = self.getOrCreateEntryBlock() let capturedArgRangeStart = clonedEntryBlock.arguments.count @@ -908,8 +909,8 @@ extension SpecializationCloner { } } - if self.context.needFixStackNesting { - self.context.fixStackNesting(in: self.cloned) + if (self.context.needFixStackNesting) { + self.context.fixStackNesting(in: targetFunction) } } } @@ -1425,12 +1426,10 @@ private struct PullbackClosureInfo { } func specializedCalleeName(_ context: FunctionPassContext) -> String { - let closureArgs = Array(self.closureArgDescriptors.map { $0.closure }) - let closureIndices = Array(self.closureArgDescriptors.map { $0.closureArgIndex }) - - return context.mangle( - withClosureArguments: closureArgs, closureArgIndices: closureIndices, - from: pullbackFn) + let closureArgs = Array(self.closureArgDescriptors.map { + (argumentIndex: $0.closureArgIndex, argumentValue: $0.closure) + }) + return context.mangle(withClosureArguments: closureArgs, from: pullbackFn) } } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift index 3d862fff6d2f1..6f181f1339d95 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift @@ -271,7 +271,7 @@ private indirect enum GlobalInitValue { /// Creates SIL for this global init value in the initializer of the `global`. func materialize(into global: GlobalVariable, from function: Function, _ context: FunctionPassContext) { - var cloner = StaticInitCloner(cloneTo: global, context) + var cloner = Cloner(cloneToGlobal: global, context) defer { cloner.deinitialize() } let builder = Builder(staticInitializerOf: global, context) @@ -280,7 +280,7 @@ private indirect enum GlobalInitValue { private func materializeRecursively( type: Type, - _ cloner: inout StaticInitCloner, + _ cloner: inout Cloner, _ builder: Builder, _ function: Function ) -> Value { @@ -289,7 +289,7 @@ private indirect enum GlobalInitValue { fatalError("cannot materialize undefined init value") case .constant(let value): - return cloner.clone(value) + return cloner.cloneRecursivelyToGlobal(value: value) case .aggregate(let fields): if type.isStruct { diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift new file mode 100644 index 0000000000000..d4e6340d7a2b8 --- /dev/null +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LoopInvariantCodeMotion.swift @@ -0,0 +1,1126 @@ +//===--- LoopInvariantCodeMotion.swift ------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SIL + +/// Hoist loop invariant code out of innermost loops. +let loopInvariantCodeMotionPass = FunctionPass(name: "loop-invariant-code-motion") { function, context in + for loop in context.loopTree.loops { + optimizeTopLevelLoop(topLevelLoop: loop, context) + } +} + +private func optimizeTopLevelLoop(topLevelLoop: Loop, _ context: FunctionPassContext) { + var innerLoops = Stack(context) + defer { innerLoops.deinitialize() } + + getWorkList(forLoop: topLevelLoop, workList: &innerLoops) + + while let thisLoop = innerLoops.pop() { + // We only support Loops with a preheader. + guard thisLoop.preheader != nil else { + continue + } + + var thisLoopChanged = false + + repeat { + var movableInstructions = analyzeLoopAndSplitLoads(loop: thisLoop, context) + thisLoopChanged = optimizeLoop(loop: thisLoop, movableInstructions: &movableInstructions, context) + } while thisLoopChanged + } +} + +/// Creates post-order DFS work list of inner loops. +private func getWorkList(forLoop loop: Loop, workList: inout Stack) { + workList.push(loop) + + for innerLoop in loop.innerLoops { + getWorkList(forLoop: innerLoop, workList: &workList) + } +} + +/// Instructions that can be moved outside the loop. +private struct MovableInstructions { + var loadAndStoreAccessPaths: [AccessPath] = [] + + var speculativelyHoistable: [Instruction] = [] + var loadsAndStores: [Instruction] = [] + var hoistUp: [Instruction] = [] + var sinkDown: [Instruction] = [] + var scopedInsts: [Instruction] = [] +} + +/// Analyzed instructions inside the currently processed loop. +private struct AnalyzedInstructions { + /// Side effects of the loop. + var loopSideEffects: StackWithCount + + private var blockSideEffectBottomMarker: StackWithCount.Marker + + /// Side effects of the currently analyzed block. + var sideEffectsOfCurrentBlock: StackWithCount.Segment { + return StackWithCount.Segment( + in: loopSideEffects, + low: blockSideEffectBottomMarker, + high: loopSideEffects.top + ) + } + + /// Contains either: + /// * an apply to the addressor of the global + /// * a builtin "once" of the global initializer + var globalInitCalls: Stack + var readOnlyApplies: Stack + var loads: Stack + var stores: Stack + var beginAccesses: Stack + var fullApplies: Stack + + /// `true` if the loop has instructions which (may) read from memory, which are not in `Loads` and not in `sideEffects`. + var hasOtherMemReadingInsts = false + + /// `true` if one of the side effects might release. + lazy var sideEffectsMayRelease = loopSideEffects.contains(where: { $0.mayRelease }) + + init (_ context: FunctionPassContext) { + self.loopSideEffects = StackWithCount(context) + self.blockSideEffectBottomMarker = loopSideEffects.top + + self.globalInitCalls = Stack(context) + self.readOnlyApplies = Stack(context) + self.loads = Stack(context) + self.stores = Stack(context) + self.beginAccesses = Stack(context) + self.fullApplies = Stack(context) + } + + mutating func deinitialize() { + readOnlyApplies.deinitialize() + globalInitCalls.deinitialize() + loopSideEffects.deinitialize() + loads.deinitialize() + stores.deinitialize() + beginAccesses.deinitialize() + fullApplies.deinitialize() + } + + /// Mark the start of currently processed block side effects. + mutating func markBeginOfBlock() { + blockSideEffectBottomMarker = loopSideEffects.top + } +} + +/// Analyzes `loop` for hosting/sinking potential. +/// Computes `MovableInstructions` we may be able to move out of the loop +/// +/// This may split some loads into smaller loads. +private func analyzeLoopAndSplitLoads(loop: Loop, _ context: FunctionPassContext) -> MovableInstructions { + // TODO: Remove once uses lowered OSSA. + loop.splitCriticalExitingAndBackEdges(context) + + var movableInstructions = MovableInstructions() + var analyzedInstructions = AnalyzedInstructions(context) + defer { analyzedInstructions.deinitialize() } + + analyzeInstructions(in: loop, &analyzedInstructions, &movableInstructions, context) + + collectHoistableReadOnlyApplies(analyzedInstructions, &movableInstructions, context) + + collectHoistableGlobalInitCalls(in: loop, analyzedInstructions, &movableInstructions, context) + + collectProjectableAccessPathsAndSplitLoads(in: loop, &analyzedInstructions, &movableInstructions, context) + + collectMovableInstructions(in: loop, &analyzedInstructions, &movableInstructions, context) + + return movableInstructions +} + +/// Analyze instructions inside the `loop`. Compute side effects and populate `analyzedInstructions`. +/// +/// - note: Ideally, `movableInstructions` should be fully computed in `collectMovableInstructions`. +private func analyzeInstructions( + in loop: Loop, + _ analyzedInstructions: inout AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + for bb in loop.loopBlocks { + analyzedInstructions.markBeginOfBlock() + + for inst in bb.instructions { + // TODO: Remove once support for values with ownership implemented. + if inst.hasOwnershipOperandsOrResults { + analyzedInstructions.analyzeSideEffects(ofInst: inst) + + // Collect fullApplies to be checked in analyzeBeginAccess + if let fullApply = inst as? FullApplySite { + analyzedInstructions.fullApplies.append(fullApply) + } + + continue + } + + switch inst { + case is FixLifetimeInst: + break // We can ignore the side effects of FixLifetimes + case let loadInst as LoadInst: + analyzedInstructions.loads.append(loadInst) + case let storeInst as StoreInst: + switch storeInst.storeOwnership { + case .assign, .initialize: + continue // TODO: Add support + case .unqualified, .trivial: + break + } + analyzedInstructions.stores.append(storeInst) + analyzedInstructions.analyzeSideEffects(ofInst: storeInst) + case let beginAccessInst as BeginAccessInst: + analyzedInstructions.beginAccesses.append(beginAccessInst) + analyzedInstructions.analyzeSideEffects(ofInst: beginAccessInst) + case let refElementAddrInst as RefElementAddrInst: + movableInstructions.speculativelyHoistable.append(refElementAddrInst) + case let condFailInst as CondFailInst: + analyzedInstructions.analyzeSideEffects(ofInst: condFailInst) + case let applyInst as ApplyInst: + if applyInst.isSafeReadOnlyApply(context.calleeAnalysis) { + analyzedInstructions.readOnlyApplies.append(applyInst) + } else if let callee = applyInst.referencedFunction, + callee.isGlobalInitFunction, // Calls to global inits are different because we don't care about side effects which are "after" the call in the loop. + !applyInst.globalInitMayConflictWith( + blockSideEffectSegment: analyzedInstructions.sideEffectsOfCurrentBlock, + context.aliasAnalysis + ) { + // Check against side-effects within the same block. + // Side-effects in other blocks are checked later (after we + // scanned all blocks of the loop) in `collectHoistableGlobalInitCalls`. + analyzedInstructions.globalInitCalls.append(applyInst) + } + + // Check for array semantics and side effects - same as default + fallthrough + default: + switch inst { + case let fullApply as FullApplySite: + analyzedInstructions.fullApplies.append(fullApply) + case let builtinInst as BuiltinInst: + switch builtinInst.id { + case .Once, .OnceWithContext: + if !builtinInst.globalInitMayConflictWith( + blockSideEffectSegment: analyzedInstructions.sideEffectsOfCurrentBlock, + context.aliasAnalysis + ) { + analyzedInstructions.globalInitCalls.append(builtinInst) + } + default: break + } + default: break + } + + analyzedInstructions.analyzeSideEffects(ofInst: inst) + + if inst.canBeHoisted(outOf: loop, context) { + movableInstructions.hoistUp.append(inst) + } + } + } + } +} + +/// Process collected read only applies. Moves them to `hoistUp` if they don't conflict with any side effects. +private func collectHoistableReadOnlyApplies( + _ analyzedInstructions: AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + var counter = 0 + for readOnlyApply in analyzedInstructions.readOnlyApplies { + // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. + if counter * analyzedInstructions.loopSideEffects.count < 8000, + readOnlyApply.isSafeReadOnlyApply( + for: analyzedInstructions.loopSideEffects, + context.aliasAnalysis, + context.calleeAnalysis + ) { + movableInstructions.hoistUp.append(readOnlyApply) + } + counter += 1 + } +} + +/// Process collected global init calls. Moves them to `hoistUp` if they don't conflict with any side effects. +private func collectHoistableGlobalInitCalls( + in loop: Loop, + _ analyzedInstructions: AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + for globalInitCall in analyzedInstructions.globalInitCalls { + // Check against side effects which are "before" (i.e. post-dominated by) the global initializer call. + // + // The effects in the same block have already been checked before + // adding this global init call to `analyzedInstructions.globalInitCalls` in `analyzeInstructions`. + if globalInitCall.parentBlock.postDominates(loop.preheader!, context.postDominatorTree), + !globalInitCall.globalInitMayConflictWith( + loopSideEffects: analyzedInstructions.loopSideEffects, + context.aliasAnalysis, + context.postDominatorTree + ) { + movableInstructions.hoistUp.append(globalInitCall) + } + } +} + +/// Collect memory locations for which we can move all loads and stores out of the loop. +/// `loads` may mutate during this loop. +private func collectProjectableAccessPathsAndSplitLoads( + in loop: Loop, + _ analyzedInstructions: inout AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + if !analyzedInstructions.hasOtherMemReadingInsts { + for storeInst in analyzedInstructions.stores { + let accessPath = storeInst.destination.accessPath + if accessPath.isLoopInvariant(loop: loop), + analyzedInstructions.isOnlyLoadedAndStored( + accessPath: accessPath, + storeAddr: storeInst.destination, + context.aliasAnalysis + ), + !movableInstructions.loadAndStoreAccessPaths.contains(accessPath), + // This is not a requirement for functional correctness, but we don't want to + // _speculatively_ load and store the value (outside of the loop). + analyzedInstructions.storesCommonlyDominateExits(of: loop, storingTo: accessPath, context), + analyzedInstructions.splitLoads( + storeAddr: storeInst.destination, + accessPath: accessPath, + context + ) { + movableInstructions.loadAndStoreAccessPaths.append(accessPath) + } + } + } +} + +/// Computes movable instructions using computed analyzed instructions. +private func collectMovableInstructions( + in loop: Loop, + _ analyzedInstructions: inout AnalyzedInstructions, + _ movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) { + var loadInstCounter = 0 + for bb in loop.loopBlocks { + for inst in bb.instructions { + // TODO: Remove once support for values with ownership implemented. + if inst.hasOwnershipOperandsOrResults { + continue + } + + switch inst { + case let fixLifetimeInst as FixLifetimeInst: + guard fixLifetimeInst.parentBlock.dominates(loop.preheader!, context.dominatorTree) else { + continue + } + + if !analyzedInstructions.sideEffectsMayRelease || !analyzedInstructions.loopSideEffectsMayWriteTo(address: fixLifetimeInst.operand.value, context.aliasAnalysis) { + movableInstructions.sinkDown.append(fixLifetimeInst) + } + case let loadInst as LoadInst: + // Avoid quadratic complexity in corner cases. Usually, this limit will not be exceeded. + if loadInstCounter * analyzedInstructions.loopSideEffects.count < 8000, + !analyzedInstructions.loopSideEffectsMayWriteTo(address: loadInst.operand.value, context.aliasAnalysis), + loadInst.loadOwnership != .take, loadInst.loadOwnership != .copy { // TODO: Add support for take and copy loads. + movableInstructions.hoistUp.append(loadInst) + } + + loadInstCounter += 1 + + movableInstructions.loadsAndStores.append(loadInst) + case let storeInst as StoreInst: + switch storeInst.storeOwnership { + case .assign, .initialize: + continue // TODO: Add support + case .unqualified, .trivial: + break + } + movableInstructions.loadsAndStores.append(storeInst) + case let condFailInst as CondFailInst: + // We can (and must) hoist cond_fail instructions if the operand is + // invariant. We must hoist them so that we preserve memory safety. A + // cond_fail that would have protected (executed before) a memory access + // must - after hoisting - also be executed before said access. + movableInstructions.hoistUp.append(condFailInst) + case let beginAccessInst as BeginAccessInst: + if beginAccessInst.canBeHoisted(outOf: loop, analyzedInstructions: analyzedInstructions, context) { + movableInstructions.scopedInsts.append(beginAccessInst) + } + default: + break + } + } + } +} + +/// Optimizes the loop by performing in following order: +/// - speculative hoist +/// - projection, hoist and sink of loads and stores +/// - hoist of instructions that are guaranteed to be executed +/// - sink +/// - hoist with sink of scoped instructions +private func optimizeLoop( + loop: Loop, + movableInstructions: inout MovableInstructions, + _ context: FunctionPassContext +) -> Bool { + var changed = false + + // TODO: If we hoist tuple_element_addr and struct_element_addr instructions here, hoistAndSinkLoadsAndStores could converge after just one execution! + changed = movableInstructions.speculativelyHoistInstructions(outOf: loop, context) || changed + changed = movableInstructions.hoistAndSinkLoadsAndStores(outOf: loop, context) || changed + changed = movableInstructions.hoistInstructions(outOf: loop, context) || changed + changed = movableInstructions.sinkInstructions(outOf: loop, context) || changed + changed = movableInstructions.hoistWithSinkScopedInstructions(outOf: loop, context) || changed + + return changed +} + +extension BasicBlock { + func containsStoresTo(accessPath: AccessPath) -> Bool { + return instructions.contains { inst in + return inst.operands.contains { operand in + if let storeInst = operand.instruction as? StoreInst, + storeInst.destination.accessPath == accessPath { + return true + } else { + return false + } + } + } + } +} + +private extension AnalyzedInstructions { + /// Adds side effects of `inst` to the analyzed instructions. + mutating func analyzeSideEffects(ofInst inst: Instruction) { + if inst.mayHaveSideEffects { + loopSideEffects.append(inst) + } else if inst.mayReadFromMemory { + hasOtherMemReadingInsts = true + } + } + + /// Returns true if all instructions in `sideEffects` which may alias with + /// this path are either loads or stores from this path. + /// + /// `storeAddr` is only needed for AliasAnalysis until we have an interface + /// that supports `AccessPath`. + func isOnlyLoadedAndStored( + accessPath: AccessPath, + storeAddr: Value, + _ aliasAnalysis: AliasAnalysis + ) -> Bool { + if (loopSideEffects.contains { sideEffect in + switch sideEffect { + case let storeInst as StoreInst: + if storeInst.isNonInitializingStoreTo(accessPath) { + return false + } + case let loadInst as LoadInst: + if loadInst.loadsFrom(accessPath) { + return false + } + default: break + } + + // Pass the original address value until we can fix alias analysis. + return sideEffect.mayReadOrWrite(address: storeAddr, aliasAnalysis) + }) { + return false + } + + if (loads.contains { loadInst in + loadInst.mayRead(fromAddress: storeAddr, aliasAnalysis) && loadInst.loadOwnership != .take && !loadInst.overlaps(accessPath: accessPath) + }) { + return false + } + + if (stores.contains { storeInst in + storeInst.mayWrite(toAddress: storeAddr, aliasAnalysis) && !storeInst.isNonInitializingStoreTo(accessPath) + }) { + return false + } + + return true + } + + /// Returns `true` if all stores to `accessPath` commonly dominate the loop exits. + func storesCommonlyDominateExits(of loop: Loop, storingTo accessPath: AccessPath, _ context: FunctionPassContext) -> Bool { + var exitingBlocksSet = BasicBlockSet(context) + var storeBlocks = BasicBlockSet(context) + var worklist = BasicBlockWorklist(context) + defer { + exitingBlocksSet.deinitialize() + storeBlocks.deinitialize() + worklist.deinitialize() + } + + // Also a store in the pre-header dominates all exists. Although the situation + // is a bit different here: the store in the pre-header remains - it's not + // (re)moved by the LICM transformation. + // But even if the loop-stores are not dominating the loop exits, it + // makes sense to move them out of the loop if this case. When this is done, + // dead-store-elimination can then most likely eliminate the store in the + // pre-header. + // + // pre_header: + // store %v1 to %addr + // header: + // cond_br %cond, then, tail + // then: + // store %v2 to %addr // a conditional store in the loop + // br tail + // tail: + // cond_br %loop_cond, header, exit + // exit: + // + // will be transformed to + // + // pre_header: + // store %v1 to %addr // <- can be removed by DSE afterwards + // header: + // cond_br %cond, then, tail + // then: + // br tail + // tail(%phi): + // cond_br %loop_cond, header, exit + // exit: + // store %phi to %addr + // + if loop.preheader!.containsStoresTo(accessPath: accessPath) { + return true + } + + // Create a set of exiting blocks for efficient lookup later. + exitingBlocksSet.insert(contentsOf: loop.exitingBlocks) + + // Collect as many recognizable store parent blocks as possible. It's ok if not all stores are collected. + storeBlocks.insert(contentsOf: stores + .filter({ $0.destination.accessPath == accessPath }) + .map(\.parentBlock)) + + // If a store is in the loop header, we already know that it's dominating all loop exits. + if storeBlocks.contains(loop.header) { + return true + } + + // Starting from the header, check whether all stores are alive. + worklist.pushIfNotVisited(loop.header) + while let block = worklist.pop() { + if storeBlocks.contains(block) { + continue + } + + if exitingBlocksSet.contains(block), + block.successors.filter({ $0.terminator is UnreachableInst }).count != block.successors.count { + return false + } + + worklist.pushIfNotVisited(contentsOf: block.successors) + } + + return true + } + + /// Returns true if `loopSideEffects` contains any memory writes which + /// may alias with the memory `address`. + func loopSideEffectsMayWriteTo(address: Value, _ aliasAnalysis: AliasAnalysis) -> Bool { + return loopSideEffects + .contains { sideEffect in + sideEffect.mayWrite(toAddress: address, aliasAnalysis) + } + } + + /// Find all loads that contain `accessPath`. Split them into a load with + /// identical `accessPath` and a set of non-overlapping loads. Add the new + /// non-overlapping loads to `loads`. + mutating func splitLoads( + storeAddr: Value, + accessPath: AccessPath, + _ context: FunctionPassContext + ) -> Bool { + var newLoads = Stack(context) + defer { + loads.append(contentsOf: newLoads) + newLoads.deinitialize() + } + var splitCounter = 0 + + while let loadInst = loads.pop() { + // Found a load wider than the store to accessPath. + // + // SplitLoads is called for each unique access path in the loop that is + // only loaded from and stored to and this loop takes time proportional to: + // num-wide-loads x num-fields x num-loop-memops + // + // For each load wider than the store, it creates a new load for each field + // in that type. Each new load is inserted in the LoadsAndStores vector. To + // avoid super-linear behavior for large types (e.g. giant tuples), limit + // growth of new loads to an arbitrary constant factor per access path. + guard splitCounter <= 6 else { + newLoads.push(loadInst) + return false + } + + guard !loadInst.isDeleted, loadInst.operand.value.accessPath.contains(accessPath) else { + newLoads.push(loadInst) + continue + } + + if let splitLoads = loadInst.trySplit(alongPath: accessPath.projectionPath, context) { + splitCounter += splitLoads.count + newLoads.append(contentsOf: splitLoads) + } + } + + return true + } +} + +private extension MovableInstructions { + /// Hoist instructions speculatively. + /// + /// Contrary to `hoistInstructions`, it doesn't only go through instructions in blocks that dominate all exits. + mutating func speculativelyHoistInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + var changed = false + + for inst in speculativelyHoistable { + changed = inst.hoist(outOf: loop, context) || changed + } + + return changed + } + + /// Hoists and sinks loads with matching stores. Projects loads. + mutating func hoistAndSinkLoadsAndStores(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + var changed = false + + for accessPath in loadAndStoreAccessPaths { + changed = hoistAndSinkLoadAndStore(outOf: loop, accessPath: accessPath, context: context) || changed + } + + return changed + } + + /// Only hoists instructions in blocks that dominate all exit and latch blocks. + /// It doesn't hoist instructions speculatively. + mutating func hoistInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(context) + var changed = false + + for bb in dominatingBlocks { + for inst in bb.instructions where hoistUp.contains(inst) { + changed = inst.hoist(outOf: loop, context) || changed + } + } + + return changed + } + + /// Sink instructions. + mutating func sinkInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + let dominatingBlocks = loop.getBlocksThatDominateAllExitingAndLatchBlocks(context) + var changed = false + + for inst in sinkDown where dominatingBlocks.contains(inst.parentBlock) { + changed = inst.sink(outOf: loop, context) || changed + } + + return changed + } + + /// Hoist and sink scoped instructions. + mutating func hoistWithSinkScopedInstructions(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + guard !loop.hasNoExitBlocks else { + return false + } + + var changed = false + + for specialInst in scopedInsts { + guard let beginAccessInst = specialInst as? BeginAccessInst else { + continue + } + + guard specialInst.hoist(outOf: loop, context) else { + continue + } + + // We only want to sink the first end_access and erase the rest to not introduce duplicates. + var sankFirst = false + for endAccess in beginAccessInst.endAccessInstructions { + if sankFirst { + context.erase(instruction: endAccess) + } else { + sankFirst = endAccess.sink(outOf: loop, context) + } + } + + changed = true + } + + return changed + } + + private mutating func hoistAndSinkLoadAndStore( + outOf loop: Loop, + accessPath: AccessPath, + context: FunctionPassContext + ) -> Bool { + // Initially load the value in the loop pre header. + let builder = Builder(before: loop.preheader!.terminator, context) + var storeAddr: Value? + + // If there are multiple stores in a block, only the last one counts. + for case let storeInst as StoreInst in loadsAndStores where storeInst.isNonInitializingStoreTo(accessPath) { + // If a store just stores the loaded value, bail. The operand (= the load) + // will be removed later, so it cannot be used as available value. + // This corner case is surprisingly hard to handle, so we just give up. + if let srcLoadInst = storeInst.source as? LoadInst, + srcLoadInst.loadsFrom(accessPath) { + return false + } + + if storeAddr == nil { + storeAddr = storeInst.destination + } else if storeInst.destination.type != storeAddr!.type { + // This transformation assumes that the values of all stores in the loop + // must be interchangeable. It won't work if stores different types + // because of casting or payload extraction even though they have the + // same access path. + return false + } + } + + guard let storeAddr else { + return false + } + + var ssaUpdater = SSAUpdater( + function: storeAddr.parentFunction, + type: storeAddr.type.objectType, + ownership: .none, + context + ) + + // Set all stored values as available values in the ssaUpdater. + for case let storeInst as StoreInst in loadsAndStores where storeInst.isNonInitializingStoreTo(accessPath) { + ssaUpdater.addAvailableValue(storeInst.source, in: storeInst.parentBlock) + } + + var cloner = Cloner(cloneBefore: loop.preheader!.terminator, context) + defer { cloner.deinitialize() } + + guard let initialAddr = (cloner.cloneRecursively(value: storeAddr) { srcAddr, cloner in + switch srcAddr { + case is AllocStackInst, is BeginBorrowInst, is MarkDependenceInst: + return .stopCloning + default: break + } + + // Clone projections until the address dominates preheader. + if srcAddr.parentBlock.dominates(loop.preheader!, context.dominatorTree) { + cloner.recordFoldedValue(srcAddr, mappedTo: srcAddr) + return .customValue(srcAddr) + } else { + // Return nil invalid to continue cloning. + return .defaultValue + } + }) else { + return false + } + + let ownership: LoadInst.LoadOwnership = loop.preheader!.terminator.parentFunction.hasOwnership ? .trivial : .unqualified + + let initialLoad = builder.createLoad(fromAddress: initialAddr, ownership: ownership) + ssaUpdater.addAvailableValue(initialLoad, in: loop.preheader!) + + var changed = false + var currentBlock: BasicBlock? + var currentVal: Value? + + // Remove all stores and replace the loads with the current value. + // + // This loop depends on loadsAndStores being in order the instructions appear in blocks. + for inst in loadsAndStores { + let block = inst.parentBlock + + if block != currentBlock { + currentBlock = block + currentVal = nil + } + + if let storeInst = inst as? StoreInst, storeInst.isNonInitializingStoreTo(accessPath) { + currentVal = storeInst.source + context.erase(instruction: storeInst) + changed = true + continue + } + + guard let loadInst = inst as? LoadInst, + loadInst.loadsFrom(accessPath) else { + continue + } + + // If we didn't see a store in this block yet, get the current value from the ssaUpdater. + let rootVal = currentVal ?? ssaUpdater.getValue(inMiddleOf: block) + + if loadInst.operand.value.accessPath == accessPath { + loadInst.replace(with: rootVal, context) + changed = true + continue + } + + guard let projectionPath = accessPath.getProjection(to: loadInst.operand.value.accessPath) else { + continue + } + + let builder = Builder(before: loadInst, context) + let projection = rootVal.createProjection(path: projectionPath, builder: builder) + loadInst.replace(with: projection, context) + + changed = true + } + + loadsAndStores.removeAll(where: { $0.isDeleted }) + + // Store back the value at all loop exits. + for exitBlock in loop.exitBlocks { + assert(exitBlock.hasSinglePredecessor, "Exiting edge should not be critical.") + + let builder = Builder(before: exitBlock.instructions.first!, context) + + let ownership: StoreInst.StoreOwnership = exitBlock.instructions.first!.parentFunction.hasOwnership ? .trivial : .unqualified + builder.createStore( + source: ssaUpdater.getValue(inMiddleOf: exitBlock), + destination: initialAddr, + ownership: ownership + ) + changed = true + } + + // In case the value is only stored but never loaded in the loop. + if initialLoad.uses.isEmpty { + context.erase(instruction: initialLoad) + } + + return changed + } +} + +private extension Instruction { + var hasOwnershipOperandsOrResults: Bool { + guard parentFunction.hasOwnership else { return false } + + return results.contains(where: { $0.ownership != .none }) + || operands.contains(where: { $0.value.ownership != .none }) + } + + /// Returns `true` if this instruction follows the default hoisting heuristic which means it + /// is not a terminator, allocation or deallocation and either a hoistable array semantics call or doesn't have memory effects. + func canBeHoisted(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + switch self { + case is TermInst, is Allocation, is Deallocation: + return false + case is ApplyInst: + switch arraySemanticsCallKind { + case .getCount, .getCapacity: + if canHoistArraySemanticsCall(to: loop.preheader!.terminator, context) { + return true + } + case .arrayPropsIsNativeTypeChecked: + return false + default: + break + } + default: + break + } + + if memoryEffects == .noEffects { + return true + } + + return false + } + + func hoist(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + guard operands.allSatisfy({ !loop.contains(block: $0.value.parentBlock) }) else { + return false + } + + let terminator = loop.preheader!.terminator + if canHoistArraySemanticsCall(to: terminator, context) { + hoistArraySemanticsCall(before: terminator, context) + } else { + move(before: terminator, context) + } + + if let singleValueInst = self as? SingleValueInstruction, + !(self is BeginAccessInst), + let identicalInst = (loop.preheader!.instructions.first { otherInst in + return singleValueInst != otherInst && singleValueInst.isIdenticalTo(otherInst) + }) { + guard let identicalSingleValueInst = identicalInst as? SingleValueInstruction else { + return true + } + + singleValueInst.replace(with: identicalSingleValueInst, context) + } + + return true + } + + func sink(outOf loop: Loop, _ context: FunctionPassContext) -> Bool { + let exitBlocks = loop.exitBlocks + var changed = false + + for exitBlock in exitBlocks { + assert(exitBlock.hasSinglePredecessor, "Exiting edge should not be critical.") + + if changed { + copy(before: exitBlock.instructions.first!, context) + } else { + move(before: exitBlock.instructions.first!, context) + changed = true + } + } + + return changed + } + + /// Returns `true` if `sideEffect` cannot be reordered with a call to this + /// global initializer. + private func globalInitMayConflictWith( + sideEffect: Instruction, + _ aliasAnalysis: AliasAnalysis + ) -> Bool { + switch sideEffect { + case let storeInst as StoreInst: + return mayReadOrWrite(address: storeInst.destinationOperand.value, aliasAnalysis) + case let loadInst as LoadInst: + return mayWrite(toAddress: loadInst.operand.value, aliasAnalysis) + case is CondFailInst: + return false + default: + return true + } + } + + /// Returns `true` if any of the instructions in `sideEffects` cannot be + /// reordered with a call to this global initializer (which is in the same basic + /// block). + func globalInitMayConflictWith( + blockSideEffectSegment: StackWithCount.Segment, + _ aliasAnalysis: AliasAnalysis + ) -> Bool { + return blockSideEffectSegment + .contains { sideEffect in + globalInitMayConflictWith( + sideEffect: sideEffect, + aliasAnalysis + ) + } + } + + /// Returns `true` if any of the instructions in `loopSideEffects` which are + /// post-dominated by a call to this global initializer cannot be reordered with + /// the call. + func globalInitMayConflictWith( + loopSideEffects: StackWithCount, + _ aliasAnalysis: AliasAnalysis, + _ postDomTree: PostDominatorTree + ) -> Bool { + return loopSideEffects + .contains { sideEffect in + // Only check instructions in blocks which are "before" (i.e. post-dominated + // by) the block which contains the init-call. + // Instructions which are before the call in the same block have already + // been checked. + parentBlock.strictlyPostDominates(sideEffect.parentBlock, postDomTree) && + globalInitMayConflictWith(sideEffect: sideEffect, aliasAnalysis) + } + } +} + +private extension StoreInst { + /// Returns a `true` if this store is a store to `accessPath`. + func isNonInitializingStoreTo(_ accessPath: AccessPath) -> Bool { + if self.storeOwnership == .initialize { + return false + } + + return accessPath == self.destination.accessPath + } +} + +private extension LoadInst { + /// Returns `true` if this load instruction loads from `accessPath` or a + /// projected address from `accessPath`. + func loadsFrom(_ accessPath: AccessPath) -> Bool { + return accessPath.getProjection(to: self.address.accessPath)?.isMaterializable ?? false + } + + func overlaps(accessPath: AccessPath) -> Bool { + // Don't use `AccessPath.mayOverlap`. We only want definite overlap. + return accessPath.isEqualOrContains(self.operand.value.accessPath) || self.operand.value.accessPath.isEqualOrContains(accessPath) + } +} + +private extension ApplyInst { + /// Returns `true` if this apply inst could be safely hoisted. + func isSafeReadOnlyApply(_ calleeAnalysis: CalleeAnalysis) -> Bool { + guard functionConvention.results.allSatisfy({ $0.convention == .unowned }) else { + return false + } + + if let callee = referencedFunction, + callee.hasSemanticsAttribute("array.props.isNativeTypeChecked") { + return false + } + + return !calleeAnalysis.getSideEffects(ofApply: self).memory.write + } + + /// Returns `true` if the `sideEffects` contain any memory writes which + /// may alias with any memory which is read by this `ApplyInst`. + /// - Note: This function should only be called on a read-only apply! + func isSafeReadOnlyApply( + for sideEffects: StackWithCount, + _ aliasAnalysis: AliasAnalysis, + _ calleeAnalysis: CalleeAnalysis + ) -> Bool { + if calleeAnalysis.getSideEffects(ofApply: self).memory == .noEffects { + return true + } + + // Check if the memory addressed by the argument may alias any writes. + for sideEffect in sideEffects { + switch sideEffect { + case let storeInst as StoreInst: + if storeInst.storeOwnership == .assign || + mayRead(fromAddress: storeInst.destination, aliasAnalysis) { + return false + } + case let copyAddrInst as CopyAddrInst: + if !copyAddrInst.isInitializationOfDestination || + mayRead(fromAddress: copyAddrInst.destination, aliasAnalysis) { + return false + } + case is ApplyInst, is BeginApplyInst, is TryApplyInst: + if calleeAnalysis.getSideEffects(ofApply: self).memory.write { + return false + } + case is CondFailInst, is StrongRetainInst, is UnmanagedRetainValueInst, + is RetainValueInst, is StrongRetainUnownedInst, is FixLifetimeInst, + is KeyPathInst, is DeallocStackInst, is DeallocStackRefInst, + is DeallocRefInst: + break + default: + if sideEffect.mayWriteToMemory { + return false + } + } + } + + return true + } +} + +private extension BeginAccessInst { + /// Returns `true` if this begin access is safe to hoist. + func canBeHoisted( + outOf loop: Loop, + analyzedInstructions: AnalyzedInstructions, + _ context: FunctionPassContext + ) -> Bool { + guard endAccessInstructions.allSatisfy({ loop.contains(block: $0.parentBlock)}) else { + return false + } + + let areBeginAccessesSafe = analyzedInstructions.beginAccesses + .allSatisfy { otherBeginAccessInst in + guard self != otherBeginAccessInst else { return true } + + return self.accessPath.isDistinct(from: otherBeginAccessInst.accessPath) + } + + guard areBeginAccessesSafe else { return false } + + var scope = InstructionRange(begin: self, ends: endAccessInstructions, context) + defer { scope.deinitialize() } + + for fullApplyInst in analyzedInstructions.fullApplies { + guard mayWriteToMemory && fullApplyInst.mayReadOrWrite(address: address, context.aliasAnalysis) || + !mayWriteToMemory && fullApplyInst.mayWrite(toAddress: address, context.aliasAnalysis) else { + continue + } + + // After hoisting the begin/end_access the apply will be within the scope, so it must not have a conflicting access. + if !scope.contains(fullApplyInst) { + return false + } + } + + switch accessPath.base { + case .class, .global: + for sideEffect in analyzedInstructions.loopSideEffects where sideEffect.mayRelease { + // Since a class might have a deinitializer, hoisting begin/end_access pair could violate + // exclusive access if the deinitializer accesses address used by begin_access. + if !scope.contains(sideEffect) { + return false + } + } + + return true + default: + return true + } + } +} + +private extension AccessPath { + /// Returns `true` if this access path is invariant in `loop`. + func isLoopInvariant(loop: Loop) -> Bool { + switch base { + case .box(let inst as Instruction), .class(let inst as Instruction), + .index(let inst as Instruction), + .pointer(let inst as Instruction), .stack(let inst as Instruction), + .storeBorrow(let inst as Instruction), + .tail(let inst as Instruction): + if loop.contains(block: inst.parentBlock) { + return false + } + case .global, .argument: + break + case .yield(let beginApplyResult): + if loop.contains(block: beginApplyResult.parentBlock) { + return false + } + case .unidentified: + return false + } + + return projectionPath.isConstant + } +} diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift index 36ceb2a8617f1..b141812d29bc4 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift @@ -364,13 +364,13 @@ private func constructObject(of allocRef: AllocRefInstBase, inInitializerOf global: GlobalVariable, _ storesToClassFields: [StoreInst], _ storesToTailElements: [StoreInst], _ context: FunctionPassContext) { - var cloner = StaticInitCloner(cloneTo: global, context) + var cloner = Cloner(cloneToGlobal: global, context) defer { cloner.deinitialize() } // Create the initializers for the fields var objectArgs = [Value]() for store in storesToClassFields { - objectArgs.append(cloner.clone(store.source as! SingleValueInstruction)) + objectArgs.append(cloner.cloneRecursivelyToGlobal(value: store.source as! SingleValueInstruction)) } let globalBuilder = Builder(staticInitializerOf: global, context) @@ -382,7 +382,7 @@ private func constructObject(of allocRef: AllocRefInstBase, for elementIdx in 0.. Bool { + return context.bridgedPassContext.canHoistArraySemanticsCall(self.bridged, toInst.bridged) + } + + func hoistArraySemanticsCall(before toInst: Instruction, _ context: some MutatingContext) { + context.bridgedPassContext.hoistArraySemanticsCall(self.bridged, toInst.bridged) // Internally updates dom tree. + context.notifyInstructionsChanged() + } +} + +extension Cloner where Context == FunctionPassContext { + func getOrCreateEntryBlock() -> BasicBlock { + if let entryBlock = targetFunction.blocks.first { + return entryBlock + } + return targetFunction.appendNewBlock(context) + } + + func cloneFunctionBody(from originalFunction: Function, entryBlockArguments: [Value]) { + entryBlockArguments.withBridgedValues { bridgedEntryBlockArgs in + let entryBlock = getOrCreateEntryBlock() + bridged.cloneFunctionBody(originalFunction.bridged, entryBlock.bridged, bridgedEntryBlockArgs) + } + } + + func cloneFunctionBody(from originalFunction: Function) { + bridged.cloneFunctionBody(originalFunction.bridged) + } +} + +func cloneFunction(from originalFunction: Function, toEmpty targetFunction: Function, _ context: FunctionPassContext) { + var cloner = Cloner(cloneToEmptyFunction: targetFunction, context) + defer { cloner.deinitialize() } + cloner.cloneFunctionBody(from: originalFunction) +} diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift index 18620eea7e914..d74055f4a2051 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/FunctionPassContext.swift @@ -45,6 +45,11 @@ struct FunctionPassContext : MutatingContext { let bridgedPDT = bridgedPassContext.getPostDomTree() return PostDominatorTree(bridged: bridgedPDT) } + + var loopTree: LoopTree { + let bridgedLT = bridgedPassContext.getLoopTree() + return LoopTree(bridged: bridgedLT, context: self) + } func loadFunction(name: StaticString, loadCalleesRecursively: Bool) -> Function? { return name.withUTF8Buffer { (nameBuffer: UnsafeBufferPointer) in @@ -92,15 +97,11 @@ struct FunctionPassContext : MutatingContext { return String(taking: bridgedPassContext.mangleOutlinedVariable(function.bridged)) } - func mangle(withClosureArguments closureArgs: [Value], closureArgIndices: [Int], from applySiteCallee: Function) -> String { - closureArgs.withBridgedValues { bridgedClosureArgsRef in - closureArgIndices.withBridgedArrayRef{bridgedClosureArgIndicesRef in - String(taking: bridgedPassContext.mangleWithClosureArgs( - bridgedClosureArgsRef, - bridgedClosureArgIndicesRef, - applySiteCallee.bridged - )) - } + func mangle(withClosureArguments closureArgs: [(argumentIndex: Int, argumentValue: Value)], + from applySiteCallee: Function + ) -> String { + closureArgs.withBridgedArrayRef{ bridgedClosureArgs in + String(taking: bridgedPassContext.mangleWithClosureArgs(bridgedClosureArgs, applySiteCallee.bridged)) } } diff --git a/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift b/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift index c89f350c7b80d..2744559fe12d7 100644 --- a/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift +++ b/SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift @@ -106,6 +106,7 @@ private func registerSwiftPasses() { registerPass(tempLValueElimination, { tempLValueElimination.run($0) }) registerPass(generalClosureSpecialization, { generalClosureSpecialization.run($0) }) registerPass(autodiffClosureSpecialization, { autodiffClosureSpecialization.run($0) }) + registerPass(loopInvariantCodeMotionPass, { loopInvariantCodeMotionPass.run($0) }) // Instruction passes registerForSILCombine(BeginBorrowInst.self, { run(BeginBorrowInst.self, $0) }) diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt index 50dab7dcccb85..35ca4d97aae75 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/CMakeLists.txt @@ -8,7 +8,6 @@ swift_compiler_sources(Optimizer AddressUtils.swift - SpecializationCloner.swift Devirtualization.swift EscapeUtils.swift FunctionSignatureTransforms.swift @@ -18,5 +17,4 @@ swift_compiler_sources(Optimizer LocalVariableUtils.swift OptUtils.swift OwnershipLiveness.swift - StaticInitCloner.swift ) diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift index d3a9302424cfe..da4d207669eb4 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift @@ -484,6 +484,22 @@ extension Instruction { } return false } + + /// Returns true if `otherInst` is in the same block and is strictly dominated by this instruction or + /// the parent block of the instruction dominates parent block of `otherInst`. + func dominates( + _ otherInst: Instruction, + _ domTree: DominatorTree + ) -> Bool { + if parentBlock == otherInst.parentBlock { + return dominatesInSameBlock(otherInst) + } else { + return parentBlock.dominates( + otherInst.parentBlock, + domTree + ) + } + } /// If this instruction uses a (single) existential archetype, i.e. it has a type-dependent operand, /// returns the concrete type if it is known. @@ -583,36 +599,22 @@ extension StoreInst { extension LoadInst { @discardableResult func trySplit(_ context: FunctionPassContext) -> Bool { - var elements = [Value]() - let builder = Builder(before: self, context) if type.isStruct { - if (type.nominal as! StructDecl).hasUnreferenceableStorage { - return false - } - guard let fields = type.getNominalFields(in: parentFunction) else { + guard !(type.nominal as! StructDecl).hasUnreferenceableStorage, + let fields = type.getNominalFields(in: parentFunction) else { return false } - for idx in 0.. LoadOwnership { @@ -623,6 +625,70 @@ extension LoadInst { return fieldValue.type.isTrivial(in: parentFunction) ? .trivial : self.loadOwnership } } + + func trySplit( + alongPath projectionPath: SmallProjectionPath, + _ context: FunctionPassContext + ) -> [LoadInst]? { + if projectionPath.isEmpty { + return nil + } + + let (fieldKind, index, pathRemainder) = projectionPath.pop() + + var elements: [LoadInst] + + switch fieldKind { + case .structField where type.isStruct: + guard !(type.nominal as! StructDecl).hasUnreferenceableStorage, + let fields = type.getNominalFields(in: parentFunction) else { + return nil + } + + elements = splitStruct(fields: fields, context) + case .tupleField where type.isTuple: + elements = splitTuple(context) + default: + return nil + } + + if let recursiveSplitLoad = elements[index].trySplit(alongPath: pathRemainder, context) { + elements.remove(at: index) + elements += recursiveSplitLoad + } + + return elements + } + + private func splitStruct(fields: NominalFieldsArray, _ context: FunctionPassContext) -> [LoadInst] { + var elements = [LoadInst]() + let builder = Builder(before: self, context) + + for idx in 0.. [LoadInst] { + var elements = [LoadInst]() + let builder = Builder(before: self, context) + + for idx in 0.. BasicBlock { - bridged.getClonedBasicBlock(originalBlock.bridged).block - } - - func cloneFunctionBody(from originalFunction: Function, entryBlockArguments: [Value]) { - entryBlockArguments.withBridgedValues { bridgedEntryBlockArgs in - bridged.cloneFunctionBody(originalFunction.bridged, self.entryBlock.bridged, bridgedEntryBlockArgs) - } - } - - func cloneFunctionBody(from originalFunction: Function) { - bridged.cloneFunctionBody(originalFunction.bridged) - } -} diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/StaticInitCloner.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/StaticInitCloner.swift deleted file mode 100644 index dcc9853052c6a..0000000000000 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/StaticInitCloner.swift +++ /dev/null @@ -1,78 +0,0 @@ -//===--- StaticInitCloner.swift --------------------------------------------==// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import SIL -import OptimizerBridging - -/// Clones the initializer value of a GlobalVariable. -/// -/// Used to transitively clone "constant" instructions, including their operands, -/// from or to the static initializer value of a GlobalVariable. -/// -struct StaticInitCloner { - private var bridged: BridgedCloner - private let context: Context - private let cloningIntoFunction: Bool - - init(cloneTo global: GlobalVariable, _ context: Context) { - self.bridged = BridgedCloner(global.bridged, context._bridged) - self.context = context - self.cloningIntoFunction = false - } - - init(cloneBefore inst: Instruction, _ context: Context) { - self.bridged = BridgedCloner(inst.bridged, context._bridged) - self.context = context - self.cloningIntoFunction = true - } - - mutating func deinitialize() { - bridged.destroy(context._bridged) - } - - /// Transitively clones `value` including its defining instruction's operands. - mutating func clone(_ value: Value) -> Value { - - if isCloned(value: value) { - return getClonedValue(of: value) - } - - if let beginAccess = value as? BeginAccessInst { - // Skip access instructions, which might be generated for UnsafePointer globals which point to other globals. - let clonedOperand = clone(beginAccess.address) - bridged.recordFoldedValue(beginAccess.bridged, clonedOperand.bridged) - return clonedOperand - } - - let inst = value.definingInstruction! - assert(!(inst is ScopedInstruction), "global init value must not contain a scoped instruction") - - for op in inst.operands { - _ = clone(op.value) - } - - bridged.clone(inst.bridged) - let clonedValue = getClonedValue(of: value) - if cloningIntoFunction { - context.notifyInstructionChanged(clonedValue.definingInstruction!) - } - return clonedValue - } - - mutating func getClonedValue(of value: Value) -> Value { - bridged.getClonedValue(value.bridged).value - } - - func isCloned(value: Value) -> Bool { - bridged.isValueCloned(value.bridged) - } -} diff --git a/SwiftCompilerSources/Sources/SIL/BasicBlock.swift b/SwiftCompilerSources/Sources/SIL/BasicBlock.swift index dcf5bb68474ec..c543559c24489 100644 --- a/SwiftCompilerSources/Sources/SIL/BasicBlock.swift +++ b/SwiftCompilerSources/Sources/SIL/BasicBlock.swift @@ -153,6 +153,14 @@ final public class BasicBlock : CustomStringConvertible, HasShortDescription, Ha return false } } + + public func isCriticalEdge(edgeIndex: Int) -> Bool { + if terminator.successors.count <= 1 { + return false + } else { + return !terminator.successors[edgeIndex].hasSinglePredecessor + } + } } /// The list of instructions in a BasicBlock. diff --git a/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift b/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift index 026e42fac92ab..30444bdb7723b 100644 --- a/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift +++ b/SwiftCompilerSources/Sources/SIL/DataStructures/Stack.swift @@ -140,7 +140,7 @@ public struct Stack : CollectionLikeSequence { public mutating func deinitialize() { removeAll() } } -extension Stack { +public extension Stack { /// Mark a stack location for future iteration. /// /// TODO: Marker should be ~Escapable. @@ -155,7 +155,7 @@ extension Stack { let low: Marker let high: Marker - init(in stack: Stack, low: Marker, high: Marker) { + public init(in stack: Stack, low: Marker, high: Marker) { if low.slab.data == nil { assert(low.index == 0, "invalid empty stack marker") // `low == nil` and `high == nil` is a valid empty segment, @@ -173,7 +173,7 @@ extension Stack { self.high = high } - func makeIterator() -> Stack.Iterator { + public func makeIterator() -> Stack.Iterator { return Iterator(slab: low.slab, index: low.index, lastSlab: high.slab, endIndex: high.index) } @@ -219,3 +219,87 @@ extension Stack { } } } + +public struct StackWithCount : CollectionLikeSequence { + public private(set) var count = 0 + private var underlyingStack: Stack + + public typealias Iterator = Stack.Iterator + + public init(_ context: some Context) { + self.underlyingStack = Stack(context) + } + + public func makeIterator() -> Stack.Iterator { + underlyingStack.makeIterator() + } + + public var first: Element? { underlyingStack.first } + public var last: Element? { underlyingStack.last } + + public mutating func push(_ element: Element) { + count += 1 + underlyingStack.push(element) + } + + /// The same as `push` to provide an Array-like append API. + public mutating func append(_ element: Element) { push(element) } + + public mutating func append(contentsOf other: S) where S.Element == Element { + for elem in other { + append(elem) + } + } + + public var isEmpty: Bool { underlyingStack.isEmpty } + + public mutating func pop() -> Element? { + if underlyingStack.isEmpty { + return nil + } + + count -= 1 + return underlyingStack.pop() + } + + public mutating func removeAll() { + underlyingStack.removeAll() + } + + /// TODO: once we have move-only types, make this a real deinit. + public mutating func deinitialize() { removeAll() } +} + +public extension StackWithCount { + typealias Marker = Stack.Marker + + struct Segment : CollectionLikeSequence { + var underlyingSegment: Stack.Segment + + public init(in stack: StackWithCount, low: Marker, high: Marker) { + underlyingSegment = Stack.Segment(in: stack.underlyingStack, low: low, high: high) + } + + public func makeIterator() -> StackWithCount.Iterator { + return underlyingSegment.makeIterator() + } + } + + var top: Marker { underlyingStack.top } + + func assertValid(marker: Marker) { underlyingStack.assertValid(marker: marker) } + + mutating func withMarker( + _ body: (inout Stack, Marker) throws -> R) rethrows -> R { + return try underlyingStack.withMarker(body) + } + + mutating func withMarker( + pushElements body: (inout Stack) throws -> R, + withNewElements handleNewElements: ((Segment) -> ()) + ) rethrows -> R { + return try underlyingStack.withMarker(pushElements: body) { [self] segment in + handleNewElements(Segment(in: self, low: segment.low, high: segment.high)) + } + } +} diff --git a/SwiftCompilerSources/Sources/SIL/Effects.swift b/SwiftCompilerSources/Sources/SIL/Effects.swift index 5cd2ea66c19a3..311f4423c60aa 100644 --- a/SwiftCompilerSources/Sources/SIL/Effects.swift +++ b/SwiftCompilerSources/Sources/SIL/Effects.swift @@ -505,6 +505,14 @@ public struct SideEffects : CustomStringConvertible, NoReflectionChildren { /// This is true when the function (or a callee, transitively) contains a /// deinit barrier instruction. public var isDeinitBarrier: Bool + + public static var noEffects: GlobalEffects { + return GlobalEffects(memory: .noEffects, ownership: .noEffects, allocates: false, isDeinitBarrier: false) + } + + public var isOnlyReading: Bool { + return !memory.write && ownership == .noEffects && !allocates && !isDeinitBarrier + } /// When called with default arguments, it creates an "effect-free" GlobalEffects. public init(memory: Memory = Memory(read: false, write: false), @@ -643,6 +651,10 @@ public struct SideEffects : CustomStringConvertible, NoReflectionChildren { copy = copy || other.copy destroy = destroy || other.destroy } + + public static var noEffects: Ownership { + return Ownership(copy: false, destroy: false) + } public static var worstEffects: Ownership { Ownership(copy: true, destroy: true) diff --git a/SwiftCompilerSources/Sources/SIL/Instruction.swift b/SwiftCompilerSources/Sources/SIL/Instruction.swift index fd5e986ac4096..0c666bb2ec154 100644 --- a/SwiftCompilerSources/Sources/SIL/Instruction.swift +++ b/SwiftCompilerSources/Sources/SIL/Instruction.swift @@ -96,6 +96,11 @@ public class Instruction : CustomStringConvertible, Hashable { BridgedContext.moveInstructionBefore(bridged, otherInstruction.bridged) context.notifyInstructionsChanged() } + + public final func copy(before otherInstruction: Instruction, _ context: some MutatingContext) { + BridgedContext.copyInstructionBefore(bridged, otherInstruction.bridged) + context.notifyInstructionsChanged() + } public var mayTrap: Bool { false } @@ -177,6 +182,10 @@ public class Instruction : CustomStringConvertible, Hashable { public static func ==(lhs: Instruction, rhs: Instruction) -> Bool { lhs === rhs } + + public func isIdenticalTo(_ otherInst: Instruction) -> Bool { + return bridged.isIdenticalTo(otherInst.bridged) + } public func hash(into hasher: inout Hasher) { hasher.combine(ObjectIdentifier(self)) diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift b/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift index 546d5c7c30893..d0863f9b3d78d 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/AccessUtils.swift @@ -392,6 +392,11 @@ public struct AccessPath : CustomStringConvertible, Hashable { public func isEqualOrContains(_ other: AccessPath) -> Bool { return getProjection(to: other) != nil } + + /// Returns true if this access contains `other` access and is not equal. + public func contains(_ other: AccessPath) -> Bool { + return !(getProjection(to: other)?.isEmpty ?? true) + } public var materializableProjectionPath: SmallProjectionPath? { if projectionPath.isMaterializable { diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt b/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt index f39a2a89ae0e3..59ba3af5f8912 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt +++ b/SwiftCompilerSources/Sources/SIL/Utilities/CMakeLists.txt @@ -9,6 +9,7 @@ swift_compiler_sources(SIL AccessUtils.swift BorrowUtils.swift + Cloner.swift ForwardingUtils.swift PhiUpdater.swift SequenceUtilities.swift diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift new file mode 100644 index 0000000000000..ffdf7b1e94dec --- /dev/null +++ b/SwiftCompilerSources/Sources/SIL/Utilities/Cloner.swift @@ -0,0 +1,141 @@ +//===--- Cloner.swift ------------------------------------------------------==// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import OptimizerBridging +import SILBridging + +/// Clones the initializer value of a GlobalVariable. +/// +/// Used to transitively clone "constant" instructions, including their operands, +/// from or to the static initializer value of a GlobalVariable. +/// +public struct Cloner { + public var bridged: BridgedCloner + public let context: Context + + public enum GetClonedResult { + case defaultValue + case customValue(Value) + case stopCloning + } + + public enum Target { + case function(Function) + case global(GlobalVariable) + } + public let target: Target + + public init(cloneToGlobal: GlobalVariable, _ context: Context) { + self.bridged = BridgedCloner(cloneToGlobal.bridged, context._bridged) + self.context = context + self.target = .global(cloneToGlobal) + } + + public init(cloneBefore inst: Instruction, _ context: Context) { + self.bridged = BridgedCloner(inst.bridged, context._bridged) + self.context = context + self.target = .function(inst.parentFunction) + } + + public init(cloneToEmptyFunction: Function, _ context: Context) { + self.bridged = BridgedCloner(cloneToEmptyFunction.bridged, context._bridged) + self.context = context + self.target = .function(cloneToEmptyFunction) + } + + public mutating func deinitialize() { + bridged.destroy(context._bridged) + } + + public var targetFunction: Function { + guard case .function(let function) = target else { + fatalError("expected cloning into a function") + } + return function + } + + public mutating func clone(instruction: Instruction) -> Instruction { + let cloned = bridged.clone(instruction.bridged).instruction + if case .function = target { + context.notifyInstructionChanged(cloned) + context.notifyInstructionsChanged() + } + return cloned + } + + public mutating func cloneRecursivelyToGlobal(value: Value) -> Value { + guard let cloned = cloneRecursively(value: value, customGetCloned: { value, cloner in + guard let beginAccess = value as? BeginAccessInst else { + return .defaultValue + } + + // Skip access instructions, which might be generated for UnsafePointer globals which point to other globals. + let clonedOperand = cloner.cloneRecursivelyToGlobal(value: beginAccess.address) + cloner.recordFoldedValue(beginAccess, mappedTo: clonedOperand) + return .customValue(clonedOperand) + }) else { + fatalError("Clone recursively to global shouldn't bail.") + } + + return cloned + } + + /// Transitively clones `value` including its defining instruction's operands. + public mutating func cloneRecursively(value: Value, customGetCloned: (Value, inout Cloner) -> GetClonedResult) -> Value? { + if isCloned(value: value) { + return getClonedValue(of: value) + } + + switch customGetCloned(value, &self) { + case .customValue(let base): + return base + case .stopCloning: + return nil + case .defaultValue: + break + } + + guard let inst = value.definingInstruction else { + fatalError("expected instruction to clone or already cloned value") + } + + for op in inst.operands { + if cloneRecursively(value: op.value, customGetCloned: customGetCloned) == nil { + return nil + } + } + + let cloned = clone(instruction: inst) + if let svi = cloned as? SingleValueInstruction { + return svi + } else if let originalMvi = value as? MultipleValueInstructionResult { + return cloned.results[originalMvi.index] + } + fatalError("unexpected instruction kind") + } + + public mutating func getClonedValue(of originalValue: Value) -> Value { + bridged.getClonedValue(originalValue.bridged).value + } + + public func isCloned(value: Value) -> Bool { + bridged.isValueCloned(value.bridged) + } + + public func getClonedBlock(for originalBlock: BasicBlock) -> BasicBlock { + bridged.getClonedBasicBlock(originalBlock.bridged).block + } + + public func recordFoldedValue(_ origValue: Value, mappedTo mappedValue: Value) { + bridged.recordFoldedValue(origValue.bridged, mappedValue.bridged) + } +} diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift b/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift index ea94f2771e83f..cfb206bd27f66 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/SmallProjectionPath.swift @@ -534,6 +534,18 @@ public struct SmallProjectionPath : Hashable, CustomStringConvertible, NoReflect return false } } + + public var isConstant: Bool { + let (kind, _, subPath) = pop() + switch kind { + case .root: + return true + case .structField, .tupleField, .enumCase, .classField, .existential, .indexedElement: + return subPath.isConstant + default: + return false + } + } } //===----------------------------------------------------------------------===// diff --git a/include/swift/SIL/SILBridging.h b/include/swift/SIL/SILBridging.h index bf38cec201f12..cb5c82cb3c99f 100644 --- a/include/swift/SIL/SILBridging.h +++ b/include/swift/SIL/SILBridging.h @@ -60,6 +60,9 @@ class SymbolicValueBumpAllocator; class ConstExprEvaluator; class SILWitnessTable; class SILDefaultWitnessTable; +class SILLoopInfo; +class SILLoop; +class BridgedClonerImpl; class SILDebugLocation; class NominalTypeDecl; class VarDecl; @@ -75,11 +78,25 @@ class SILLocation; class BasicBlockSet; class NodeSet; class OperandSet; -class ClonerWithFixedLocation; class FixedSizeSlabPayload; class FixedSizeSlab; } +struct BridgedLoop { + swift::SILLoop * _Nonnull l; + + BRIDGED_INLINE SwiftInt getInnerLoopCount() const; + BRIDGED_INLINE BridgedLoop getInnerLoop(SwiftInt index) const; + + BRIDGED_INLINE SwiftInt getBasicBlockCount() const; + BRIDGED_INLINE BridgedBasicBlock getBasicBlock(SwiftInt index) const; + + BRIDGED_INLINE OptionalBridgedBasicBlock getPreheader() const; + BRIDGED_INLINE BridgedBasicBlock getHeader() const; + + BRIDGED_INLINE bool contains(BridgedBasicBlock block) const; +}; + bool swiftModulesInitialized(); void registerBridgedClass(BridgedStringRef className, SwiftMetatype metatype); @@ -678,6 +695,7 @@ struct BridgedInstruction { bool maySynchronize() const; bool mayBeDeinitBarrierNotConsideringSideEffects() const; BRIDGED_INLINE bool shouldBeForwarding() const; + BRIDGED_INLINE bool isIdenticalTo(BridgedInstruction inst) const; // =========================================================================// // Generalized instruction subclasses @@ -1376,18 +1394,6 @@ struct BridgedOperandSet { SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedFunction getFunction() const; }; -struct BridgedCloner { - swift::ClonerWithFixedLocation * _Nonnull cloner; - - BridgedCloner(BridgedGlobalVar var, BridgedContext context); - BridgedCloner(BridgedInstruction inst, BridgedContext context); - void destroy(BridgedContext context); - SWIFT_IMPORT_UNSAFE BridgedValue getClonedValue(BridgedValue v); - bool isValueCloned(BridgedValue v) const; - void clone(BridgedInstruction inst); - void recordFoldedValue(BridgedValue origValue, BridgedValue mappedValue); -}; - struct BridgedContext { swift::SILContext * _Nonnull context; @@ -1465,6 +1471,7 @@ struct BridgedContext { BRIDGED_INLINE void eraseInstruction(BridgedInstruction inst, bool salvageDebugInfo) const; BRIDGED_INLINE void eraseBlock(BridgedBasicBlock block) const; static BRIDGED_INLINE void moveInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst); + static BRIDGED_INLINE void copyInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst); // SSAUpdater @@ -1501,6 +1508,25 @@ struct BridgedContext { SWIFT_IMPORT_UNSAFE BRIDGED_INLINE Slab freeSlab(Slab slab) const; }; +struct BridgedCloner { + swift::BridgedClonerImpl * _Nonnull cloner; + + BridgedCloner(BridgedGlobalVar var, BridgedContext context); + BridgedCloner(BridgedInstruction inst, BridgedContext context); + BridgedCloner(BridgedFunction emptyFunction, BridgedContext context); + void destroy(BridgedContext context); + SWIFT_IMPORT_UNSAFE BridgedFunction getCloned() const; + SWIFT_IMPORT_UNSAFE BridgedBasicBlock getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const; + void cloneFunctionBody(BridgedFunction originalFunction, BridgedBasicBlock clonedEntryBlock, + BridgedValueArray clonedEntryBlockArgs) const; + void cloneFunctionBody(BridgedFunction originalFunction) const; + SWIFT_IMPORT_UNSAFE BridgedValue getClonedValue(BridgedValue v); + bool isValueCloned(BridgedValue v) const; + void recordClonedInstruction(BridgedInstruction origInst, BridgedInstruction clonedInst) const; + void recordFoldedValue(BridgedValue orig, BridgedValue mapped) const; + BridgedInstruction clone(BridgedInstruction inst); +}; + struct BridgedVerifier { typedef void (* _Nonnull VerifyFunctionFn)(BridgedContext, BridgedFunction); diff --git a/include/swift/SIL/SILBridgingImpl.h b/include/swift/SIL/SILBridgingImpl.h index 0d2cbef48701e..f465b7b1d4367 100644 --- a/include/swift/SIL/SILBridgingImpl.h +++ b/include/swift/SIL/SILBridgingImpl.h @@ -1070,6 +1070,10 @@ bool BridgedInstruction::shouldBeForwarding() const { llvm::isa(unbridged()); } +bool BridgedInstruction::isIdenticalTo(BridgedInstruction inst) const { + return unbridged()->isIdenticalTo(inst.unbridged()); +} + SwiftInt BridgedInstruction::MultipleValueInstruction_getNumResults() const { return getAs()->getNumResults(); } @@ -2940,6 +2944,10 @@ void BridgedContext::moveInstructionBefore(BridgedInstruction inst, BridgedInstr swift::SILBasicBlock::moveInstruction(inst.unbridged(), beforeInst.unbridged()); } +void BridgedContext::copyInstructionBefore(BridgedInstruction inst, BridgedInstruction beforeInst) { + inst.unbridged()->clone(beforeInst.unbridged()); +} + OptionalBridgedFunction BridgedContext::lookupStdlibFunction(BridgedStringRef name) const { return {context->lookupStdlibFunction(name.unbridged())}; } diff --git a/include/swift/SILOptimizer/Analysis/ArrayCallKind.h b/include/swift/SILOptimizer/Analysis/ArrayCallKind.h new file mode 100644 index 0000000000000..f3db620d0d162 --- /dev/null +++ b/include/swift/SILOptimizer/Analysis/ArrayCallKind.h @@ -0,0 +1,45 @@ +//===--- ArrayCallKind.h -------------------------------------- -*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef ARRAY_CALL_KIND_H +#define ARRAY_CALL_KIND_H + +/// The kind of array operation identified by looking at the semantics attribute +/// of the called function. +enum class ArrayCallKind { + kNone = 0, + kArrayPropsIsNativeTypeChecked, + kCheckSubscript, + kCheckIndex, + kGetCount, + kGetCapacity, + kGetElement, + kGetElementAddress, + kMakeMutable, + kEndMutation, + kMutateUnknown, + kReserveCapacityForAppend, + kWithUnsafeMutableBufferPointer, + kAppendContentsOf, + kAppendElement, + // The following two semantic function kinds return the result @owned + // instead of operating on self passed as parameter. If you are adding + // a function, and it has a self parameter, make sure that it is defined + // before this comment. + kArrayInit, + kArrayInitEmpty, + kArrayUninitialized, + kArrayUninitializedIntrinsic, + kArrayFinalizeIntrinsic +}; + +#endif diff --git a/include/swift/SILOptimizer/Analysis/ArraySemantic.h b/include/swift/SILOptimizer/Analysis/ArraySemantic.h index 7bbca9d7c7e50..0b144562dea94 100644 --- a/include/swift/SILOptimizer/Analysis/ArraySemantic.h +++ b/include/swift/SILOptimizer/Analysis/ArraySemantic.h @@ -14,40 +14,12 @@ #define SWIFT_SILOPTIMIZER_ANALYSIS_ARRAYSEMANTIC_H #include "swift/SIL/SILInstruction.h" +#include "swift/SILOptimizer/Analysis/ArrayCallKind.h" namespace swift { class DominanceInfo; -/// The kind of array operation identified by looking at the semantics attribute -/// of the called function. -enum class ArrayCallKind { - kNone = 0, - kArrayPropsIsNativeTypeChecked, - kCheckSubscript, - kCheckIndex, - kGetCount, - kGetCapacity, - kGetElement, - kGetElementAddress, - kMakeMutable, - kEndMutation, - kMutateUnknown, - kReserveCapacityForAppend, - kWithUnsafeMutableBufferPointer, - kAppendContentsOf, - kAppendElement, - // The following two semantic function kinds return the result @owned - // instead of operating on self passed as parameter. If you are adding - // a function, and it has a self parameter, make sure that it is defined - // before this comment. - kArrayInit, - kArrayInitEmpty, - kArrayUninitialized, - kArrayUninitializedIntrinsic, - kArrayFinalizeIntrinsic -}; - /// Return true is the given function is an array semantics call. ArrayCallKind getArraySemanticsKind(SILFunction *f); diff --git a/include/swift/SILOptimizer/OptimizerBridging.h b/include/swift/SILOptimizer/OptimizerBridging.h index efab99a89aef6..e553c96cdb755 100644 --- a/include/swift/SILOptimizer/OptimizerBridging.h +++ b/include/swift/SILOptimizer/OptimizerBridging.h @@ -19,6 +19,7 @@ /// See include guidelines and caveats in `BasicBridging.h`. #include "swift/AST/ASTBridging.h" #include "swift/SIL/SILBridging.h" +#include "swift/SILOptimizer/Analysis/ArrayCallKind.h" #ifndef NOT_COMPILED_WITH_SWIFT_PURE_BRIDGING_MODE @@ -37,11 +38,14 @@ SWIFT_BEGIN_NULLABILITY_ANNOTATIONS namespace swift { class AliasAnalysis; +class ArraySemanticsCall; class BasicCalleeAnalysis; class CalleeList; class DeadEndBlocks; class DominanceInfo; class PostDominanceInfo; +class SILLoopInfo; +class SILLoop; class SwiftPassInvocation; class SILVTable; class SpecializationCloner; @@ -111,6 +115,8 @@ struct BridgedDomTree { swift::DominanceInfo * _Nonnull di; BRIDGED_INLINE bool dominates(BridgedBasicBlock dominating, BridgedBasicBlock dominated) const; + BRIDGED_INLINE SwiftInt getNumberOfChildren(BridgedBasicBlock bb) const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedBasicBlock getChildAt(BridgedBasicBlock bb, SwiftInt index) const; }; struct BridgedPostDomTree { @@ -119,15 +125,13 @@ struct BridgedPostDomTree { BRIDGED_INLINE bool postDominates(BridgedBasicBlock dominating, BridgedBasicBlock dominated) const; }; -struct BridgedSpecializationCloner { - swift::SpecializationCloner * _Nonnull cloner; - - SWIFT_IMPORT_UNSAFE BridgedSpecializationCloner(BridgedFunction emptySpecializedFunction); - SWIFT_IMPORT_UNSAFE BridgedFunction getCloned() const; - SWIFT_IMPORT_UNSAFE BridgedBasicBlock getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const; - void cloneFunctionBody(BridgedFunction originalFunction, BridgedBasicBlock clonedEntryBlock, - BridgedValueArray clonedEntryBlockArgs) const; - void cloneFunctionBody(BridgedFunction originalFunction) const; +struct BridgedLoopTree { + swift::SILLoopInfo * _Nonnull li; + + BRIDGED_INLINE SwiftInt getTopLevelLoopCount() const; + BRIDGED_INLINE BridgedLoop getLoop(SwiftInt index) const; + + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedBasicBlock splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const; }; struct BridgedPassContext { @@ -146,6 +150,16 @@ struct BridgedPassContext { SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedDeadEndBlocksAnalysis getDeadEndBlocksAnalysis() const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedDomTree getDomTree() const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedPostDomTree getPostDomTree() const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedDeclObj getSwiftArrayDecl() const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedDeclObj getSwiftMutableSpanDecl() const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedLoopTree getLoopTree() const; + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedLoop getLoop() const; + + // Array semantics call + + static BRIDGED_INLINE ArrayCallKind getArraySemanticsCallKind(BridgedInstruction inst); + BRIDGED_INLINE bool canHoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction toInst) const; + BRIDGED_INLINE void hoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction beforeInst) const; // AST @@ -173,8 +187,7 @@ struct BridgedPassContext { BridgedOwnedString mangleOutlinedVariable(BridgedFunction function) const; BridgedOwnedString mangleAsyncRemoved(BridgedFunction function) const; BridgedOwnedString mangleWithDeadArgs(BridgedArrayRef bridgedDeadArgIndices, BridgedFunction function) const; - BridgedOwnedString mangleWithClosureArgs(BridgedValueArray closureArgs, - BridgedArrayRef closureArgIndices, + BridgedOwnedString mangleWithClosureArgs(BridgedArrayRef closureArgIndices, BridgedFunction applySiteCallee) const; BridgedOwnedString mangleWithBoxToStackPromotedArgs(BridgedArrayRef bridgedPromotedArgIndices, BridgedFunction bridgedOriginalFunction) const; diff --git a/include/swift/SILOptimizer/OptimizerBridgingImpl.h b/include/swift/SILOptimizer/OptimizerBridgingImpl.h index 91f560c440360..f700122cd64cd 100644 --- a/include/swift/SILOptimizer/OptimizerBridgingImpl.h +++ b/include/swift/SILOptimizer/OptimizerBridgingImpl.h @@ -21,9 +21,11 @@ #include "swift/Demangling/Demangle.h" #include "swift/SILOptimizer/Analysis/AliasAnalysis.h" +#include "swift/SILOptimizer/Analysis/ArraySemantic.h" #include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h" #include "swift/SILOptimizer/Analysis/DeadEndBlocksAnalysis.h" #include "swift/SILOptimizer/Analysis/DominanceAnalysis.h" +#include "swift/SILOptimizer/Analysis/LoopAnalysis.h" #include "swift/SILOptimizer/OptimizerBridging.h" #include "swift/SILOptimizer/PassManager/PassManager.h" #include "swift/SILOptimizer/PassManager/Transforms.h" @@ -83,10 +85,62 @@ bool BridgedDomTree::dominates(BridgedBasicBlock dominating, BridgedBasicBlock d return di->dominates(dominating.unbridged(), dominated.unbridged()); } +SwiftInt BridgedDomTree::getNumberOfChildren(BridgedBasicBlock bb) const { + return di->getNode(bb.unbridged())->getNumChildren(); +} + +BridgedBasicBlock BridgedDomTree::getChildAt(BridgedBasicBlock bb, SwiftInt index) const { + return {di->getNode(bb.unbridged())->begin()[index]->getBlock()}; +} + bool BridgedPostDomTree::postDominates(BridgedBasicBlock dominating, BridgedBasicBlock dominated) const { return pdi->dominates(dominating.unbridged(), dominated.unbridged()); } +//===----------------------------------------------------------------------===// +// BridgedLoopTree, BridgedLoop +//===----------------------------------------------------------------------===// + +SwiftInt BridgedLoopTree::getTopLevelLoopCount() const { + return li->end() - li->begin(); +} + +BridgedLoop BridgedLoopTree::getLoop(SwiftInt index) const { + return {li->begin()[index]}; +} + +BridgedBasicBlock BridgedLoopTree::splitEdge(BridgedBasicBlock bb, SwiftInt edgeIndex, BridgedDomTree domTree) const { + return {swift::splitEdge(bb.unbridged()->getTerminator(), edgeIndex, domTree.di, li)}; +} + +SwiftInt BridgedLoop::getInnerLoopCount() const { + return l->end() - l->begin(); +} + +BridgedLoop BridgedLoop::getInnerLoop(SwiftInt index) const { + return {l->begin()[index]}; +} + +SwiftInt BridgedLoop::getBasicBlockCount() const { + return l->getBlocks().size(); +} + +BridgedBasicBlock BridgedLoop::getBasicBlock(SwiftInt index) const { + return {l->getBlocks()[index]}; +} + +OptionalBridgedBasicBlock BridgedLoop::getPreheader() const { + return {l->getLoopPreheader()}; +} + +BridgedBasicBlock BridgedLoop::getHeader() const { + return {l->getHeader()}; +} + +bool BridgedLoop::contains(BridgedBasicBlock block) const { + return l->contains(block.unbridged()); +} + //===----------------------------------------------------------------------===// // BridgedPassContext //===----------------------------------------------------------------------===// @@ -136,6 +190,38 @@ BridgedPostDomTree BridgedPassContext::getPostDomTree() const { return {pda->get(invocation->getFunction())}; } +BridgedLoopTree BridgedPassContext::getLoopTree() const { + auto *lt = invocation->getPassManager()->getAnalysis(); + return {lt->get(invocation->getFunction())}; +} + +BridgedDeclObj BridgedPassContext::getSwiftArrayDecl() const { + swift::SILModule *mod = invocation->getPassManager()->getModule(); + return {mod->getASTContext().getArrayDecl()}; +} + +BridgedDeclObj BridgedPassContext::getSwiftMutableSpanDecl() const { + swift::SILModule *mod = invocation->getPassManager()->getModule(); + return {mod->getASTContext().getMutableSpanDecl()}; +} + +// Array semantics call + +ArrayCallKind BridgedPassContext::getArraySemanticsCallKind(BridgedInstruction inst) { + swift::ArraySemanticsCall semCall(inst.unbridged()); + return semCall.getKind(); +} + +bool BridgedPassContext::canHoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction toInst) const { + swift::ArraySemanticsCall semCall(inst.unbridged()); + return semCall.canHoist(toInst.unbridged(), getDomTree().di); +} + +void BridgedPassContext::hoistArraySemanticsCall(BridgedInstruction inst, BridgedInstruction beforeInst) const { + swift::ArraySemanticsCall semCall(inst.unbridged()); + semCall.hoist(beforeInst.unbridged(), getDomTree().di); +} + // AST SWIFT_IMPORT_UNSAFE BRIDGED_INLINE diff --git a/include/swift/SILOptimizer/PassManager/Passes.def b/include/swift/SILOptimizer/PassManager/Passes.def index a87327b70711a..a68a0ac379c53 100644 --- a/include/swift/SILOptimizer/PassManager/Passes.def +++ b/include/swift/SILOptimizer/PassManager/Passes.def @@ -148,6 +148,8 @@ PASS(TempRValueElimination, "temp-rvalue-elimination", "Remove short-lived immutable temporary copies") PASS(TempLValueElimination, "temp-lvalue-elimination", "Remove short-lived immutable temporary l-values") +PASS(LoopInvariantCodeMotion, "loop-invariant-code-motion", + "New Loop Invariant Code Motion") // NOTE - ExperimentalSwiftBasedClosureSpecialization and AutodiffClosureSpecialization are a WIP PASS(ExperimentalSwiftBasedClosureSpecialization, "experimental-swift-based-closure-specialization", diff --git a/lib/SIL/Utils/SILBridging.cpp b/lib/SIL/Utils/SILBridging.cpp index 75eb2eec85244..81f59168328e8 100644 --- a/lib/SIL/Utils/SILBridging.cpp +++ b/lib/SIL/Utils/SILBridging.cpp @@ -546,59 +546,96 @@ BridgedInstruction BridgedBuilder::createSwitchEnumAddrInst(BridgedValue enumAdd } //===----------------------------------------------------------------------===// -// BridgedCloner +// BridgedCloner //===----------------------------------------------------------------------===// // Need to put ClonerWithFixedLocation into namespace swift to forward reference -// it in SILBridging.h. +// it in OptimizerBridging.h. namespace swift { -class ClonerWithFixedLocation : public SILCloner { - friend class SILInstructionVisitor; - friend class SILCloner; +class BridgedClonerImpl : public SILCloner { + friend class SILInstructionVisitor; + friend class SILCloner; - SILDebugLocation insertLoc; + bool hasFixedLocation; + union { + SILDebugLocation fixedLocation; + ScopeCloner scopeCloner; + }; -public: - ClonerWithFixedLocation(SILGlobalVariable *gVar) - : SILCloner(gVar), - insertLoc(ArtificialUnreachableLocation(), nullptr) {} + SILInstruction *result = nullptr; - ClonerWithFixedLocation(SILInstruction *insertionPoint) - : SILCloner(*insertionPoint->getFunction()), - insertLoc(insertionPoint->getDebugLocation()) { +public: + BridgedClonerImpl(SILGlobalVariable *gVar) + : SILCloner(gVar), + hasFixedLocation(true), + fixedLocation(ArtificialUnreachableLocation(), nullptr) {} + + BridgedClonerImpl(SILInstruction *insertionPoint) + : SILCloner(*insertionPoint->getFunction()), + hasFixedLocation(true), + fixedLocation(insertionPoint->getDebugLocation()) { Builder.setInsertionPoint(insertionPoint); } + BridgedClonerImpl(SILFunction &emptyFunction) + : SILCloner(emptyFunction), + hasFixedLocation(false), + scopeCloner(ScopeCloner(emptyFunction)) {} + + ~BridgedClonerImpl() { + if (hasFixedLocation) { + fixedLocation.~SILDebugLocation(); + } else { + scopeCloner.~ScopeCloner(); + } + } + SILValue getClonedValue(SILValue v) { return getMappedValue(v); } - void cloneInst(SILInstruction *inst) { + SILInstruction *cloneInst(SILInstruction *inst) { + result = nullptr; visit(inst); + ASSERT(result && "instruction not cloned"); + return result; } -protected: - SILLocation remapLocation(SILLocation loc) { - return insertLoc.getLocation(); + if (hasFixedLocation) + return fixedLocation.getLocation(); + return loc; } const SILDebugScope *remapScope(const SILDebugScope *DS) { - return insertLoc.getScope(); + if (hasFixedLocation) + return fixedLocation.getScope(); + return scopeCloner.getOrCreateClonedScope(DS); + } + + void postProcess(SILInstruction *Orig, SILInstruction *Cloned) { + result = Cloned; + SILCloner::postProcess(Orig, Cloned); } + }; } // namespace swift BridgedCloner::BridgedCloner(BridgedGlobalVar var, BridgedContext context) - : cloner(new ClonerWithFixedLocation(var.getGlobal())) { + : cloner(new BridgedClonerImpl(var.getGlobal())) { context.context->notifyNewCloner(); } BridgedCloner::BridgedCloner(BridgedInstruction inst, BridgedContext context) - : cloner(new ClonerWithFixedLocation(inst.unbridged())) { + : cloner(new BridgedClonerImpl(inst.unbridged())) { + context.context->notifyNewCloner(); +} + +BridgedCloner::BridgedCloner(BridgedFunction emptyFunction, BridgedContext context) + : cloner(new BridgedClonerImpl(*emptyFunction.getFunction())) { context.context->notifyNewCloner(); } @@ -608,6 +645,10 @@ void BridgedCloner::destroy(BridgedContext context) { context.context->notifyClonerDestroyed(); } +BridgedFunction BridgedCloner::getCloned() const { + return { &cloner->getBuilder().getFunction() }; +} + BridgedValue BridgedCloner::getClonedValue(BridgedValue v) { return {cloner->getClonedValue(v.getSILValue())}; } @@ -616,12 +657,32 @@ bool BridgedCloner::isValueCloned(BridgedValue v) const { return cloner->isValueCloned(v.getSILValue()); } -void BridgedCloner::clone(BridgedInstruction inst) { - cloner->cloneInst(inst.unbridged()); +void BridgedCloner::recordClonedInstruction(BridgedInstruction origInst, BridgedInstruction clonedInst) const { + cloner->recordClonedInstruction(origInst.unbridged(), clonedInst.unbridged()); +} + +void BridgedCloner::recordFoldedValue(BridgedValue orig, BridgedValue mapped) const { + cloner->recordFoldedValue(orig.getSILValue(), mapped.getSILValue()); +} + +BridgedInstruction BridgedCloner::clone(BridgedInstruction inst) { + return {cloner->cloneInst(inst.unbridged())->asSILNode()}; +} + +BridgedBasicBlock BridgedCloner::getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const { + return { cloner->getOpBasicBlock(originalBasicBlock.unbridged()) }; +} + +void BridgedCloner::cloneFunctionBody(BridgedFunction originalFunction, + BridgedBasicBlock clonedEntryBlock, + BridgedValueArray clonedEntryBlockArgs) const { + llvm::SmallVector clonedEntryBlockArgsStorage; + auto clonedEntryBlockArgsArrayRef = clonedEntryBlockArgs.getValues(clonedEntryBlockArgsStorage); + cloner->cloneFunctionBody(originalFunction.getFunction(), clonedEntryBlock.unbridged(), clonedEntryBlockArgsArrayRef); } -void BridgedCloner::recordFoldedValue(BridgedValue origValue, BridgedValue mappedValue) { - cloner->recordFoldedValue(origValue.getSILValue(), mappedValue.getSILValue()); +void BridgedCloner::cloneFunctionBody(BridgedFunction originalFunction) const { + cloner->cloneFunction(originalFunction.getFunction()); } //===----------------------------------------------------------------------===// diff --git a/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp b/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp index 0b12222f412b0..b9a1314a2fd2e 100644 --- a/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp +++ b/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp @@ -337,7 +337,7 @@ void ArrayInfo::classifyUsesOfArray(SILValue arrayValue) { if (arrayOp.doesNotChangeArray()) continue; - if (arrayOp.getKind() == swift::ArrayCallKind::kArrayFinalizeIntrinsic) { + if (arrayOp.getKind() == ArrayCallKind::kArrayFinalizeIntrinsic) { classifyUsesOfArray((ApplyInst *)arrayOp); continue; } diff --git a/lib/SILOptimizer/LoopTransforms/LICM.cpp b/lib/SILOptimizer/LoopTransforms/LICM.cpp index 567990e355560..ceba90e5cf9cd 100644 --- a/lib/SILOptimizer/LoopTransforms/LICM.cpp +++ b/lib/SILOptimizer/LoopTransforms/LICM.cpp @@ -1486,6 +1486,7 @@ hoistLoadsAndStores(AccessPath accessPath, SILLoop *loop) { continue; } auto loadWithAccess = isLoadWithinAccess(I, accessPath); + if (!loadWithAccess) { continue; } diff --git a/lib/SILOptimizer/PassManager/PassPipeline.cpp b/lib/SILOptimizer/PassManager/PassPipeline.cpp index 8448ad1e217a8..7782243d14198 100644 --- a/lib/SILOptimizer/PassManager/PassPipeline.cpp +++ b/lib/SILOptimizer/PassManager/PassPipeline.cpp @@ -392,7 +392,7 @@ void addHighLevelLoopOptPasses(SILPassPipelinePlan &P) { // before CanonicalOSSA re-hoists destroys. P.addAccessEnforcementReleaseSinking(); P.addAccessEnforcementOpts(); - P.addHighLevelLICM(); + P.addLoopInvariantCodeMotion(); // Simplify CFG after LICM that creates new exit blocks P.addSimplifyCFG(); // LICM might have added new merging potential by hoisting @@ -481,7 +481,7 @@ void addFunctionPasses(SILPassPipelinePlan &P, // late as possible before inlining because it must run between runs of the // inliner when the pipeline restarts. if (OpLevel == OptimizationLevelKind::MidLevel) { - P.addHighLevelLICM(); + P.addLoopInvariantCodeMotion(); P.addArrayCountPropagation(); P.addBoundsCheckOpts(); P.addDCE(); @@ -739,10 +739,10 @@ static void addMidLevelFunctionPipeline(SILPassPipelinePlan &P) { // A LICM pass at mid-level is mainly needed to hoist addressors of globals. // It needs to be before global_init functions are inlined. - P.addLICM(); + P.addLoopInvariantCodeMotion(); // Run loop unrolling after inlining and constant propagation, because loop // trip counts may have became constant. - P.addLICM(); + P.addLoopInvariantCodeMotion(); P.addLoopUnroll(); } @@ -848,7 +848,7 @@ static void addLateLoopOptPassPipeline(SILPassPipelinePlan &P) { // It will also set the no_nested_conflict for dynamic accesses P.addAccessEnforcementReleaseSinking(); P.addAccessEnforcementOpts(); - P.addLICM(); + P.addLoopInvariantCodeMotion(); P.addCOWOpts(); // Simplify CFG after LICM that creates new exit blocks P.addSimplifyCFG(); @@ -892,7 +892,7 @@ static void addLastChanceOptPassPipeline(SILPassPipelinePlan &P) { P.addAccessEnforcementDom(); // addAccessEnforcementDom might provide potential for LICM: // A loop might have only one dynamic access now, i.e. hoistable - P.addLICM(); + P.addLoopInvariantCodeMotion(); // Verify AccessStorage once again after optimizing and lowering OSSA. #ifndef NDEBUG diff --git a/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp b/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp index 2257f526f0b9c..b96f862b23b63 100644 --- a/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp +++ b/lib/SILOptimizer/Transforms/DeadObjectElimination.cpp @@ -686,7 +686,7 @@ recursivelyCollectInteriorUses(ValueBase *DefInst, continue; } ArraySemanticsCall AS(svi); - if (AS.getKind() == swift::ArrayCallKind::kArrayFinalizeIntrinsic) { + if (AS.getKind() == ArrayCallKind::kArrayFinalizeIntrinsic) { if (!recursivelyCollectInteriorUses(svi, AddressNode, IsInteriorAddress)) return false; continue; diff --git a/lib/SILOptimizer/Utils/OptimizerBridging.cpp b/lib/SILOptimizer/Utils/OptimizerBridging.cpp index a41d498448100..181eb1c13a70a 100644 --- a/lib/SILOptimizer/Utils/OptimizerBridging.cpp +++ b/lib/SILOptimizer/Utils/OptimizerBridging.cpp @@ -272,32 +272,31 @@ BridgedOwnedString BridgedPassContext::mangleWithDeadArgs(BridgedArrayRef bridge } BridgedOwnedString BridgedPassContext::mangleWithClosureArgs( - BridgedValueArray bridgedClosureArgs, - BridgedArrayRef bridgedClosureArgIndices, - BridgedFunction applySiteCallee + BridgedArrayRef bridgedClosureArgs, BridgedFunction applySiteCallee ) const { + + struct ClosureArgElement { + SwiftInt argIdx; + BridgeValueExistential argValue; + }; + auto pass = Demangle::SpecializationPass::ClosureSpecializer; auto serializedKind = applySiteCallee.getFunction()->getSerializedKind(); Mangle::FunctionSignatureSpecializationMangler mangler(applySiteCallee.getFunction()->getASTContext(), pass, serializedKind, applySiteCallee.getFunction()); - llvm::SmallVector closureArgsStorage; - auto closureArgs = bridgedClosureArgs.getValues(closureArgsStorage); - auto closureArgIndices = bridgedClosureArgIndices.unbridged(); - - assert(closureArgs.size() == closureArgIndices.size() && - "Number of closures arguments and number of closure indices do not match!"); + auto closureArgs = bridgedClosureArgs.unbridged(); - for (size_t i = 0; i < closureArgs.size(); i++) { - auto closureArg = closureArgs[i]; - auto closureArgIndex = closureArgIndices[i]; + for (ClosureArgElement argElmt : closureArgs) { + auto closureArg = argElmt.argValue.value.getSILValue(); + auto closureArgIndex = argElmt.argIdx; if (auto *PAI = dyn_cast(closureArg)) { mangler.setArgumentClosureProp(closureArgIndex, const_cast(PAI)); } else { auto *TTTFI = cast(closureArg); - mangler.setArgumentClosureProp(closureArgIndex, + mangler.setArgumentClosureProp(closureArgIndex, const_cast(TTTFI)); } } @@ -459,43 +458,12 @@ BridgedCalleeAnalysis::CalleeList BridgedCalleeAnalysis::getDestructors(BridgedT return ca->getDestructors(type.unbridged(), isExactType); } -namespace swift { - class SpecializationCloner: public SILClonerWithScopes { - friend class SILInstructionVisitor; - friend class SILCloner; - public: - using SuperTy = SILClonerWithScopes; - SpecializationCloner(SILFunction &emptySpecializedFunction): SuperTy(emptySpecializedFunction) {} - }; -} // namespace swift - -BridgedSpecializationCloner::BridgedSpecializationCloner(BridgedFunction emptySpecializedFunction): - cloner(new SpecializationCloner(*emptySpecializedFunction.getFunction())) {} - -BridgedFunction BridgedSpecializationCloner::getCloned() const { - return { &cloner->getBuilder().getFunction() }; -} - -BridgedBasicBlock BridgedSpecializationCloner::getClonedBasicBlock(BridgedBasicBlock originalBasicBlock) const { - return { cloner->getOpBasicBlock(originalBasicBlock.unbridged()) }; -} - -void BridgedSpecializationCloner::cloneFunctionBody(BridgedFunction originalFunction, BridgedBasicBlock clonedEntryBlock, BridgedValueArray clonedEntryBlockArgs) const { - llvm::SmallVector clonedEntryBlockArgsStorage; - auto clonedEntryBlockArgsArrayRef = clonedEntryBlockArgs.getValues(clonedEntryBlockArgsStorage); - cloner->cloneFunctionBody(originalFunction.getFunction(), clonedEntryBlock.unbridged(), clonedEntryBlockArgsArrayRef); -} - -void BridgedSpecializationCloner::cloneFunctionBody(BridgedFunction originalFunction) const { - cloner->cloneFunction(originalFunction.getFunction()); -} - void BridgedBuilder::destroyCapturedArgs(BridgedInstruction partialApply) const { if (auto *pai = llvm::dyn_cast(partialApply.unbridged()); pai->isOnStack()) { auto b = unbridged(); - return swift::insertDestroyOfCapturedArguments(pai, b); + return swift::insertDestroyOfCapturedArguments(pai, b); } else { - assert(false && "`destroyCapturedArgs` must only be called on a `partial_apply` on stack!"); + assert(false && "`destroyCapturedArgs` must only be called on a `partial_apply` on stack!"); } } diff --git a/test/Interop/Cxx/function/default-arguments-multifile.swift b/test/Interop/Cxx/function/default-arguments-multifile.swift index 2dd3bc315d7b0..eb103118778a2 100644 --- a/test/Interop/Cxx/function/default-arguments-multifile.swift +++ b/test/Interop/Cxx/function/default-arguments-multifile.swift @@ -3,12 +3,12 @@ // RUN: mkdir -p %t/artifacts // Multiple usages in the same module. -// RUN: %target-build-swift %t/main.swift %t/b.swift %t/c.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -o %t/artifacts/out +// RUN: %target-build-swift -save-temps %t/main.swift %t/b.swift %t/c.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -o %t/artifacts/out // RUN: %empty-directory(%t/artifacts) // Multiple usages across different modules. -// RUN: %target-build-swift -emit-library -module-name BarLibrary -emit-module -emit-module-path %t/artifacts/BarLibrary.swiftmodule %t/b.swift %t/c.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -o %t/artifacts/%target-library-name(BarLibrary) -// RUN: %target-build-swift %t/uses-library.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -I %t/artifacts -L %t/artifacts -lBarLibrary -o %t/artifacts/uses-library +// RUN: %target-build-swift -save-temps -emit-library -module-name BarLibrary -emit-module -emit-module-path %t/artifacts/BarLibrary.swiftmodule %t/b.swift %t/c.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -o %t/artifacts/%target-library-name(BarLibrary) +// RUN: %target-build-swift -save-temps %t/uses-library.swift -cxx-interoperability-mode=upcoming-swift -I %S/Inputs -I %t/artifacts -L %t/artifacts -lBarLibrary -o %t/artifacts/uses-library // FIXME: Windows test can be enabled once merge-modules step is removed from the old driver, // or the Windows CI starts to use the new driver to compile the compiler. diff --git a/test/SILOptimizer/inlinearray_bounds_check_tests.swift b/test/SILOptimizer/inlinearray_bounds_check_tests.swift index 49e75f2340379..d586e3cc2ab06 100644 --- a/test/SILOptimizer/inlinearray_bounds_check_tests.swift +++ b/test/SILOptimizer/inlinearray_bounds_check_tests.swift @@ -262,7 +262,7 @@ public func inlinearray_binary_search_spl(_ v: InlineArray, // This prevents LoopRotate which prevent bounds checks opts since it depends on induction variable analysis which doesn't work on unrotated loops. // CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A35_sum_iterate_to_count_with_trap_splySis11InlineArrayVy$63_SiGF : // CHECK-SIL: bb2 -// CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" +// CHECK-NOT-SIL: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_br // CHECK-SIL-LABEL: } // end sil function '$s30inlinearray_bounds_check_tests0A35_sum_iterate_to_count_with_trap_splySis11InlineArrayVy$63_SiGF' public func inlinearray_sum_iterate_to_count_with_trap_spl(_ v: InlineArray<64, Int>) -> Int { @@ -326,7 +326,7 @@ public func inlinearray_inc_by_one(_ v: inout InlineArray, _ // CHECK-SIL-LABEL: sil @$s30inlinearray_bounds_check_tests0A15_inc_by_one_splyys11InlineArrayVy$63_SiGz_SitF : // CHECK-SIL: bb2 -// CHECK-SIL: cond_fail {{.*}}, "Index out of bounds" +// CHECK-NOT-SIL: cond_fail {{.*}}, "Index out of bounds" // CHECK-SIL: cond_br // CHECK-SIL-LABEL: } // end sil function '$s30inlinearray_bounds_check_tests0A15_inc_by_one_splyys11InlineArrayVy$63_SiGz_SitF' public func inlinearray_inc_by_one_spl(_ v: inout InlineArray<64, Int>, _ n: Int) { diff --git a/test/SILOptimizer/licm.sil b/test/SILOptimizer/licm.sil index 0dbf019064002..404aea3119a38 100644 --- a/test/SILOptimizer/licm.sil +++ b/test/SILOptimizer/licm.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=none -enable-sil-verify-all %s -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=none -enable-sil-verify-all %s -loop-invariant-code-motion | %FileCheck %s // REQUIRES: swift_in_compiler @@ -182,13 +182,14 @@ bb4: sil [_semantics "array.get_count"] @getCount : $@convention(method) (@guaranteed Array) -> Int sil @user : $@convention(thin) (Int) -> () -// CHECK-LABEL: sil @dont_hoist_get_count_on_low_level_sil -// CHECK: {{^}}bb1: +// CHECK-LABEL: sil @hoist_get_count_on_low_level_sil +// CHECK: {{^}}bb0(%0 : $Array): // CHECK: apply +// CHECK: {{^}}bb1: // CHECK: apply // CHECK: {{^}}bb2: // CHECK: return -sil @dont_hoist_get_count_on_low_level_sil : $@convention(thin) (@guaranteed Array) -> () { +sil @hoist_get_count_on_low_level_sil : $@convention(thin) (@guaranteed Array) -> () { bb0(%0 : $Array): br bb1 @@ -943,12 +944,9 @@ bb5: return %99 : $() } -// Test load splitting with a loop-invariant stored value. The loop +// Test load splitting with a loop-invariant. The loop // will be empty after combined load/store hoisting/sinking. // -// TODO: sink a struct_extract (or other non-side-effect instructions) -// with no uses in the loop. -// // CHECK-LABEL: sil shared @testLoadSplit : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { // CHECK: [[PRELOAD:%.*]] = load %{{.*}} : $*Int64 // CHECK: [[STOREDVAL:%.*]] = struct_extract %0 : $Int64, #Int64._value @@ -984,12 +982,12 @@ bb2: bb3: %result = tuple (%val1 : $Index, %val2 : $Int64, %val3 : $Builtin.Int64) return %result : $(Index, Int64, Builtin.Int64) -} +} // end sil function 'testLoadSplit' // Test load splitting with a loop-varying stored value. // CHECK-LABEL: sil shared @testLoadSplitPhi : $@convention(method) (Int64, Builtin.RawPointer) -> (Index, Int64, Builtin.Int64) { // CHECK: [[PRELOAD:%.*]] = load %{{.*}} : $*Int64 -// CHECK: br bb1(%4 : $Int64) +// CHECK: br bb1(%{{.*}} : $Int64) // CHECK: bb1([[PHI:%.*]] : $Int64): // CHECK-NEXT: [[OUTERVAL:%.*]] = struct $Index ([[PHI]] : $Int64) // CHECK-NEXT: [[EXTRACT:%.*]] = struct_extract [[PHI]] : $Int64, #Int64._value @@ -1030,7 +1028,7 @@ bb2: bb3: %result = tuple (%outerVal : $Index, %middleVal : $Int64, %innerVal : $Builtin.Int64) return %result : $(Index, Int64, Builtin.Int64) -} +} // end sil function 'testLoadSplitPhi' struct State { @_hasStorage var valueSet: (Int64, Int64, Int64) { get set } @@ -1045,11 +1043,12 @@ struct State { // CHECK: bb0(%0 : $Builtin.RawPointer): // CHECK: [[HOISTADR:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 0 // ...Preload stored element #1 +// CHECK: tuple_element_addr // CHECK: [[PRELOADADR:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 1 // CHECK: [[PRELOAD:%.*]] = load [[PRELOADADR]] : $*Int64 // ...Split element 0 -// CHECK: [[SPLIT0:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 0 -// CHECK: [[ELT0:%.*]] = load [[SPLIT0]] : $*Int64 +// CHECK: [[ELT0:%.*]] = load [[HOISTADR]] : $*Int64 +// CHECK: [[HOISTVAL:%.*]] = struct_extract [[ELT0]] : $Int64, #Int64._value // ...Split element 2 // CHECK: [[SPLIT2:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64, Int64), 2 // CHECK: [[ELT2:%.*]] = load [[SPLIT2]] : $*Int64 @@ -1057,8 +1056,6 @@ struct State { // CHECK: [[SINGLEADR:%.*]] = struct_element_addr %{{.*}} : $*State, #State.singleValue // CHECK: [[SINGLEVAL:%.*]] = load [[SINGLEADR]] : $*Int64 // ...Hoisted element 0 -// CHECK: [[HOISTLOAD:%.*]] = load [[HOISTADR]] : $*Int64 -// CHECK: [[HOISTVAL:%.*]] = struct_extract [[HOISTLOAD]] : $Int64, #Int64._value // CHECK: br bb1([[PRELOAD]] : $Int64) // ...Loop // CHECK: bb1([[PHI:%.*]] : $Int64): @@ -1166,7 +1163,6 @@ bb3: // CHECK: [[ELT_1a:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, (Int64, Int64)), 1 // CHECK: [[ELT_1_0:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64), 0 // CHECK: [[V_1_0:%.*]] = load [[ELT_1_0]] : $*Int64 -// CHECK: [[ELT_1b:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, (Int64, Int64)), 1 // CHECK: [[ELT_1_1:%.*]] = tuple_element_addr %{{.*}} : $*(Int64, Int64), 1 // CHECK: [[V_1_1:%.*]] = load [[ELT_1_1]] : $*Int64 // CHECK: br bb1([[V_0:%.*]] : $Int64, [[V_1_0]] : $Int64) @@ -1241,25 +1237,24 @@ bb3: return %val1 : $(Int64, (Int64, Int64)) } -// Two wide loads. The first can be successfully split and the second -// half hoisted. The second cannot be split because of a pointer -// cast. Make sure two remaining loads and the store are still in the loop. +// Split two wide loads. // // CHECK-LABEL: sil hidden @testSplitNonStandardProjection : $@convention(method) (Int64, Builtin.RawPointer) -> ((Int64, (Int64, Int64)), (Int64, Int64)) { // CHECK: bb0(%0 : $Int64, %1 : $Builtin.RawPointer): // -// The first load was split, so one half is hoisted. -// CHECK: [[V1:%.*]] = load %{{.*}} : $*Int64 -// CHECK: br bb1 -// CHECK: bb1: -// CHECK: [[V0:%.*]] = load %{{.*}} : $*Int64 -// CHECK: [[INNER:%.*]] = tuple ([[V0]] : $Int64, [[V1]] : $Int64) -// CHECK: store %0 to %{{.*}} : $*Int64 -// CHECK: [[OUTER:%.*]] = load %{{.*}} : $*(Int64, (Int64, Int64)) +// CHECK: [[ADDR1:%.*]] = tuple_element_addr %{{.*}}, 0 +// CHECK: [[V1:%.*]] = load [[ADDR1]] : $*Int64 +// CHECK: [[ADDR0:%.*]] = tuple_element_addr %{{.*}}, 1 +// CHECK: [[V0:%.*]] = load [[ADDR0]] : $*Int64 +// CHECK: [[OUTER:%.*]] = tuple (%{{.*}} : $Int64, %{{.*}} : $(Int64, Int64)) +// CHECK: br bb1([[V1]] : $Int64) +// CHECK: bb1([[PHI:%.*]] : $Int64): +// CHECK: [[INNER:%.*]] = tuple ([[PHI]] : $Int64, [[V0]] : $Int64) // CHECK: cond_br undef, bb2, bb3 // CHECK: bb2: -// CHECK: br bb1 +// CHECK: br bb1(%0 : $Int64) // CHECK: bb3: +// CHECK: store %0 to [[ADDR1]] : $*Int64 // CHECK: [[RESULT:%.*]] = tuple ([[OUTER]] : $(Int64, (Int64, Int64)), [[INNER]] : $(Int64, Int64)) // CHECK: return [[RESULT]] : $((Int64, (Int64, Int64)), (Int64, Int64)) // CHECK-LABEL: } // end sil function 'testSplitNonStandardProjection' @@ -1291,11 +1286,11 @@ bb3: // CHECK: bb0(%0 : $Int64, %1 : $Int64, %2 : $Builtin.RawPointer): // CHECK: [[ELT_1:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 1 // CHECK: [[V1:%.*]] = load %4 : $*(Int64, Int64) -// CHECK: [[ELT_0:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 0 -// CHECK: [[V0:%.*]] = load %6 : $*Int64 // CHECK: [[ARG0:%.*]] = tuple (%0 : $Int64, %0 : $Int64) -// CHECK: [[ARG0_0:%.*]] = tuple_extract %8 : $(Int64, Int64), 0 // CHECK: [[ARG1:%.*]] = tuple (%1 : $Int64, %1 : $Int64) +// CHECK: [[ELT_0:%.*]] = tuple_element_addr %3 : $*(Int64, (Int64, Int64)), 0 +// CHECK: [[V0:%.*]] = load %9 : $*Int64 +// CHECK: [[ARG0_0:%.*]] = tuple_extract [[ARG0]] : $(Int64, Int64), 0 // CHECK: br bb1([[V1]] : $(Int64, Int64)) // CHECK: bb1([[PHI:%.*]] : $(Int64, Int64)): // CHECK: [[LOOPVAL:%.*]] = tuple ([[V0]] : $Int64, [[PHI]] : $(Int64, Int64)) @@ -1343,8 +1338,10 @@ class C {} // This won't be hoisted because we can't find a base to check if it is invariant // CHECK-LABEL: sil @testLoopInvariantStoreNoBase1 : +// CHECK: bb3(%11 : $Builtin.RawPointer): +// CHECK-NOT: load // CHECK: bb6: -// CHECK-NOT: store +// CHECK: store // CHECK-LABEL: } // end sil function 'testLoopInvariantStoreNoBase1' sil @testLoopInvariantStoreNoBase1 : $@convention(thin) (Builtin.BridgeObject, Double) -> () { bb0(%0 : $Builtin.BridgeObject, %1 : $Double): @@ -1380,8 +1377,10 @@ bb6: // This won't be hoisted because we can't find a base to check if it is invariant // CHECK-LABEL: sil @testLoopInvariantStoreNoBase2 : +// CHECK: bb3(%11 : $Builtin.RawPointer): +// CHECK-NOT: load // CHECK: bb6: -// CHECK-NOT: store +// CHECK: store // CHECK-LABEL: } // end sil function 'testLoopInvariantStoreNoBase2' sil @testLoopInvariantStoreNoBase2 : $@convention(thin) (Builtin.BridgeObject, Double) -> () { bb0(%0 : $Builtin.BridgeObject, %1 : $Double): diff --git a/test/SILOptimizer/licm_apply.sil b/test/SILOptimizer/licm_apply.sil index df5a5381a2731..77fe0bf6b5a4f 100644 --- a/test/SILOptimizer/licm_apply.sil +++ b/test/SILOptimizer/licm_apply.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -compute-side-effects -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -compute-side-effects -loop-invariant-code-motion | %FileCheck %s // REQUIRES: swift_in_compiler diff --git a/test/SILOptimizer/licm_exclusivity.sil b/test/SILOptimizer/licm_exclusivity.sil index bd7049cc7bc06..fcfe9770af4be 100644 --- a/test/SILOptimizer/licm_exclusivity.sil +++ b/test/SILOptimizer/licm_exclusivity.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=checked -enable-sil-verify-all %s -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enforce-exclusivity=checked -enable-sil-verify-all %s -loop-invariant-code-motion | %FileCheck %s sil_stage canonical diff --git a/test/SILOptimizer/licm_exclusivity.swift b/test/SILOptimizer/licm_exclusivity.swift index 458a57c42e0d8..0b9f56e6bd889 100644 --- a/test/SILOptimizer/licm_exclusivity.swift +++ b/test/SILOptimizer/licm_exclusivity.swift @@ -1,17 +1,9 @@ -// RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -Xllvm -debug-only=sil-licm -primary-file %s 2>&1 | %FileCheck %s --check-prefix=TESTLICM -// RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -Xllvm -debug-only=sil-licm -primary-file %s 2>&1 | %FileCheck %s --check-prefix=TESTLICM2 // RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -primary-file %s | %FileCheck %s --check-prefix=TESTSIL -// RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -Xllvm -debug-only=sil-licm -whole-module-optimization %s 2>&1 | %FileCheck %s --check-prefix=TESTLICMWMO // RUN: %target-swift-frontend -O -enforce-exclusivity=checked -Xllvm -sil-print-types -emit-sil -whole-module-optimization %s | %FileCheck %s --check-prefix=TESTSILWMO // REQUIRES: optimized_stdlib,asserts,swift_stdlib_no_asserts // REQUIRES: PTRSIZE=64 -// TESTLICM-LABEL: Processing loops in {{.*}}run_ReversedArray{{.*}} -// TESTLICM: Hoist and Sink pairs attempt -// TESTLICM: Hoisted -// TESTLICM: Successfully hoisted and sank pair - // TESTSIL-LABEL: sil hidden @$s16licm_exclusivity17run_ReversedArrayyySiF : $@convention(thin) (Int) -> () { // TESTSIL: bb // TESTSIL: begin_access [modify] [dynamic] [no_nested_conflict] @@ -35,10 +27,6 @@ func run_ReversedArray(_ N: Int) { } } -// TESTLICM2-LABEL: Processing loops in {{.*}}count_unicodeScalars{{.*}} -// TESTLICM2: Hoist and Sink pairs attempt -// TESTLICM2: Hoisted - // TESTSIL-LABEL: sil @$s16licm_exclusivity20count_unicodeScalarsyySS17UnicodeScalarViewVF : $@convention(thin) (@guaranteed String.UnicodeScalarView) -> () { // TESTSIL: bb0(%0 : $String.UnicodeScalarView) // TESTSIL: bb5: @@ -66,12 +54,6 @@ public class ClassWithArrs { B = [Int](repeating: 0, count: N) } -// TESTLICMWMO-LABEL: Processing loops in {{.*}}ClassWithArrsC7readArr{{.*}} -// TESTLICMWMO: Hoist and Sink pairs attempt -// TESTLICMWMO: Hoisted -// TESTLICMWMO: Successfully hoisted and sank pair -// TESTLICMWMO: Hoisted -// TESTLICMWMO: Successfully hoisted and sank pair // TESTSILWMO-LABEL: sil {{.*}}@$s16licm_exclusivity13ClassWithArrsC7readArryyF : $@convention(method) (@guaranteed ClassWithArrs) -> () { // TESTSILWMO: [[R1:%.*]] = ref_element_addr %0 : $ClassWithArrs, #ClassWithArrs.A // TESTSILWMO: [[R2:%.*]] = ref_element_addr %0 : $ClassWithArrs, #ClassWithArrs.B diff --git a/test/SILOptimizer/licm_multiend.sil b/test/SILOptimizer/licm_multiend.sil index 32d83b3d14ad6..5a22419f66b5f 100644 --- a/test/SILOptimizer/licm_multiend.sil +++ b/test/SILOptimizer/licm_multiend.sil @@ -1,4 +1,4 @@ -// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -licm | %FileCheck %s +// RUN: %target-sil-opt -sil-print-types -enable-sil-verify-all %s -loop-invariant-code-motion | %FileCheck %s // REQUIRES: PTRSIZE=64 // REQUIRES: OS=macosx