diff --git a/stdlib/public/Atomics/AtomicInteger.swift b/stdlib/public/Atomics/AtomicInteger.swift new file mode 100644 index 0000000000000..b53f43463fea3 --- /dev/null +++ b/stdlib/public/Atomics/AtomicInteger.swift @@ -0,0 +1,106 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +public protocol AtomicInteger: AtomicValue, FixedWidthInteger +where _AtomicStorage: _PrimitiveAtomicInteger {} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +public protocol _PrimitiveAtomicInteger: _PrimitiveAtomic { + /// Perform an atomic wrapping increment operation on the value referenced by + /// `pointer` and return the original value, applying the specified memory + /// ordering. + /// + /// Note: This operation silently wraps around on overflow, like the + /// `&+=` operator does on integer values. + /// + /// - Parameter operand: The value to add to the current value. + /// - Parameter pointer: A memory location previously initialized with a value + /// returned by `prepareAtomicStorage(for:)`. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The original value before the operation. + @_semantics("atomics.requires_constant_orderings") + static func _atomicLoadThenWrappingIncrement( + by operand: Self, + at pointer: UnsafeMutablePointer, + ordering: AtomicUpdateOrdering + ) -> Self + + /// Perform an atomic wrapping decrement operation on the value referenced by + /// `pointer` and return the original value, applying the specified memory + /// ordering. + /// + /// Note: This operation silently wraps around on overflow, like the + /// `&-=` operator does on integer values. + /// + /// - Parameter operand: The value to subtract from the current value. + /// - Parameter pointer: A memory location previously initialized with a value + /// returned by `prepareAtomicStorage(for:)`. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The original value before the operation. + @_semantics("atomics.requires_constant_orderings") + static func _atomicLoadThenWrappingDecrement( + by operand: Self, + at pointer: UnsafeMutablePointer, + ordering: AtomicUpdateOrdering + ) -> Self + + /// Perform an atomic bitwise AND operation on the value referenced by + /// `pointer` and return the original value, applying the specified memory + /// ordering. + /// + /// - Parameter operand: An integer value. + /// - Parameter pointer: A memory location previously initialized with a value + /// returned by `prepareAtomicStorage(for:)`. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The original value before the operation. + @_semantics("atomics.requires_constant_orderings") + static func _atomicLoadThenBitwiseAnd( + with operand: Self, + at pointer: UnsafeMutablePointer, + ordering: AtomicUpdateOrdering + ) -> Self + + /// Perform an atomic bitwise OR operation on the value referenced by + /// `pointer` and return the original value, applying the specified memory + /// ordering. + /// + /// - Parameter operand: An integer value. + /// - Parameter pointer: A memory location previously initialized with a value + /// returned by `prepareAtomicStorage(for:)`. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The original value before the operation. + @_semantics("atomics.requires_constant_orderings") + static func _atomicLoadThenBitwiseOr( + with operand: Self, + at pointer: UnsafeMutablePointer, + ordering: AtomicUpdateOrdering + ) -> Self + + /// Perform an atomic bitwise XOR operation on the value referenced by + /// `pointer` and return the original value, applying the specified memory + /// ordering. + /// + /// - Parameter operand: An integer value. + /// - Parameter pointer: A memory location previously initialized with a value + /// returned by `prepareAtomicStorage(for:)`. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The original value before the operation. + @_semantics("atomics.requires_constant_orderings") + static func _atomicLoadThenBitwiseXor( + with operand: Self, + at pointer: UnsafeMutablePointer, + ordering: AtomicUpdateOrdering + ) -> Self +} diff --git a/stdlib/public/Atomics/AtomicIntegerConformances.swift.gyb b/stdlib/public/Atomics/AtomicIntegerConformances.swift.gyb new file mode 100644 index 0000000000000..72b9003e70097 --- /dev/null +++ b/stdlib/public/Atomics/AtomicIntegerConformances.swift.gyb @@ -0,0 +1,241 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +%{ + def atomicTypes(): + return [ + # Swift Builtin Getter + ("Int", "Word", "_builtinWordValue"), + ("UInt", "Word", "_builtinWordValue"), + ("Int64", "Int64", "_value"), + ("UInt64", "Int64", "_value"), + ("Int32", "Int32", "_value"), + ("UInt32", "Int32", "_value"), + ("Int16", "Int16", "_value"), + ("UInt16", "Int16", "_value"), + ("Int8", "Int8", "_value"), + ("UInt8", "Int8", "_value"), + ] + + def rmwOrderings(): + return [ + # Swift enum case, llvm name, failure name + ('relaxed', 'monotonic', 'monotonic'), + ('acquiring', 'acquire', 'acquire'), + ('releasing', 'release', 'monotonic'), + ('acquiringAndReleasing', 'acqrel', 'acquire'), + ('sequentiallyConsistent', 'seqcst', 'seqcst'), + ] + + def loadOrderings(): + return [ + # Swift enum case, llvm name + ('relaxed', 'monotonic'), + ('acquiring', 'acquire'), + ('sequentiallyConsistent', 'seqcst'), + ] + + def storeOrderings(): + return [ + # Swift enum case, llvm name + ('relaxed', 'monotonic'), + ('releasing', 'release'), + ('sequentiallyConsistent', 'seqcst'), + ] + + def caseStatementForOrdering(ordering): + return "case .{}".format(ordering) + + # FIXME: Swift should provide intrinsics for arbitrary ordering pairs + def llvmOrders(rmw, load): # See llvm/Support/AtomicOrdering.h + def max(rmw, load): + if load == "acquire": + if rmw == "monotonic": + return "acquire" + if rmw == "release": + return "acqrel" + if load == "seqcst": + return "seqcst" + return rmw + return max(rmw, load) + "_" + load + + def lowerFirst(str): + return str[:1].lower() + str[1:] if str else "" + + def argLabel(label): + return label + ": " if label <> "_" else "" + + integerOperations = [ + # Swift name, llvm name, operator, label, doc + ('WrappingIncrement', 'add', '&+', "by", "wrapping add"), + ('WrappingDecrement', 'sub', '&-', "by", "wrapping subtract"), + ('BitwiseAnd', 'and', '&', "with", "bitwise and"), + ('BitwiseOr', 'or', '|', "with", "bitwise or"), + ('BitwiseXor', 'xor', '^', "with", "bitwise xor") + ] +}% + +import Swift + +% for (swiftType, builtinType, getter) in atomicTypes(): +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension ${swiftType}: _PrimitiveAtomic { + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicLoad( + at pointer: UnsafeMutablePointer<${swiftType}>, + ordering: AtomicLoadOrdering + ) -> ${swiftType} { + switch ordering { +% for (swiftOrder, llvmOrder) in loadOrderings(): + ${caseStatementForOrdering(swiftOrder)}: + return ${swiftType}(Builtin.atomicload_${llvmOrder}_${builtinType}( + pointer._rawValue)) +% end + default: + Builtin.unreachable() + } + } + + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicStore( + _ desired: ${swiftType}, + at pointer: UnsafeMutablePointer<${swiftType}>, + ordering: AtomicStoreOrdering + ) { + switch ordering { +% for (swiftOrder, llvmOrder) in storeOrderings(): + ${caseStatementForOrdering(swiftOrder)}: + Builtin.atomicstore_${llvmOrder}_${builtinType}( + pointer._rawValue, + desired.${getter}) +% end + default: + Builtin.unreachable() + } + } + + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicExchange( + _ desired: ${swiftType}, + at pointer: UnsafeMutablePointer<${swiftType}>, + ordering: AtomicUpdateOrdering + ) -> ${swiftType} { + switch ordering { +% for (swiftOrder, llvmOrder, _) in rmwOrderings(): + ${caseStatementForOrdering(swiftOrder)}: + return ${swiftType}(Builtin.atomicrmw_xchg_${llvmOrder}_${builtinType}( + pointer._rawValue, desired.${getter})) +% end + default: + Builtin.unreachable() + } + } + + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicCompareExchange( + expected: ${swiftType}, + desired: ${swiftType}, + at pointer: UnsafeMutablePointer<${swiftType}>, + ordering: AtomicUpdateOrdering + ) -> (exchanged: Bool, original: ${swiftType}) { + switch ordering { +% for (swiftOrder, llvmOrder, failureOrder) in rmwOrderings(): + ${caseStatementForOrdering(swiftOrder)}: + let (oldValue, won) = Builtin.cmpxchg_${llvmOrder}_${failureOrder}_${builtinType}( + pointer._rawValue, expected.${getter}, desired.${getter}) + return (Bool(_builtinBooleanLiteral: won), ${swiftType}(oldValue)) +% end + default: + Builtin.unreachable() + } + } + + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicCompareExchange( + expected: ${swiftType}, + desired: ${swiftType}, + at pointer: UnsafeMutablePointer<${swiftType}>, + successOrdering: AtomicUpdateOrdering, + failureOrdering: AtomicLoadOrdering + ) -> (exchanged: Bool, original: ${swiftType}) { + switch (successOrdering, failureOrdering) { +% for (swiftSuccess, llvmSuccess, _) in rmwOrderings(): +% for (swiftFailure, llvmFailure) in loadOrderings(): + case (.${swiftSuccess}, .${swiftFailure}): + let (oldValue, won) = Builtin.cmpxchg_${llvmOrders(llvmSuccess, llvmFailure)}_${builtinType}( + pointer._rawValue, expected.${getter}, desired.${getter}) + return (Bool(_builtinBooleanLiteral: won), ${swiftType}(oldValue)) +% end +% end + default: + Builtin.unreachable() + } + } + + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicWeakCompareExchange( + expected: ${swiftType}, + desired: ${swiftType}, + at pointer: UnsafeMutablePointer<${swiftType}>, + successOrdering: AtomicUpdateOrdering, + failureOrdering: AtomicLoadOrdering + ) -> (exchanged: Bool, original: ${swiftType}) { + switch (successOrdering, failureOrdering) { +% for (swiftSuccess, llvmSuccess, _) in rmwOrderings(): +% for (swiftFailure, llvmFailure) in loadOrderings(): + case (.${swiftSuccess}, .${swiftFailure}): + let (oldValue, won) = Builtin.cmpxchg_${llvmOrders(llvmSuccess, llvmFailure)}_weak_${builtinType}( + pointer._rawValue, expected.${getter}, desired.${getter}) + return (Bool(_builtinBooleanLiteral: won), ${swiftType}(oldValue)) +% end +% end + default: + Builtin.unreachable() + } + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension ${swiftType}: AtomicInteger { + public typealias _AtomicStorage = ${swiftType} +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension ${swiftType}: _PrimitiveAtomicInteger { +% for (name, llvmop, op, label, doc) in integerOperations: +% defaultValue = " = 1" if label <> "" else "" + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + @discardableResult + public static func _atomicLoadThen${name}( + ${label} operand: Self${defaultValue}, + at pointer: UnsafeMutablePointer, + ordering: AtomicUpdateOrdering + ) -> Self { + switch ordering { + % for (swiftOrder, llvmOrder, _) in rmwOrderings(): + ${caseStatementForOrdering(swiftOrder)}: + return Self(Builtin.atomicrmw_${llvmop}_${llvmOrder}_${builtinType}( + pointer._rawValue, operand.${getter})) + % end + default: + Builtin.unreachable() + } + } +% end +} diff --git a/stdlib/public/Atomics/AtomicMemoryOrderings.swift b/stdlib/public/Atomics/AtomicMemoryOrderings.swift new file mode 100644 index 0000000000000..16e3d85f509d9 --- /dev/null +++ b/stdlib/public/Atomics/AtomicMemoryOrderings.swift @@ -0,0 +1,282 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +/// Specifies the memory ordering semantics of an atomic load operation. +@frozen +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +public struct AtomicLoadOrdering { + // This struct works like a non-frozen enum whose cases aren't reorderable. + + @usableFromInline + internal var _rawValue: Int + + @_semantics("constant_evaluable") + @inlinable @_transparent // Debug performance + internal init(_rawValue: Int) { + self._rawValue = _rawValue + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicLoadOrdering { + // FIXME: Explain these ordering levels in more detail. + + /// Guarantees the atomicity of the specific operation on which it is applied, + /// but imposes no ordering constraints on any other reads or writes. + @_semantics("constant_evaluable") + @_alwaysEmitIntoClient + @_transparent // Debug performance + public static var relaxed: Self { + Self(_rawValue: 0) + } + + /// An acquiring load prevents the effects of subsequent load and store + /// operations on the current thread from appearing to happen before the + /// effect of the atomic operation itself. + @_semantics("constant_evaluable") + @_alwaysEmitIntoClient + @_transparent // Debug performance + public static var acquiring: Self { + Self(_rawValue: 1) + } + + @_semantics("constant_evaluable") + @_alwaysEmitIntoClient + @_transparent // Debug performance + public static var sequentiallyConsistent: Self { + Self(_rawValue: 4) + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicLoadOrdering: Equatable { + @_transparent // Debug performance + public static func ==(left: Self, right: Self) -> Bool { + return left._rawValue == right._rawValue + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicLoadOrdering: Hashable { + @inlinable + public func hash(into hasher: inout Hasher) { + hasher.combine(_rawValue) + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicLoadOrdering: CustomStringConvertible { + public var description: String { + switch self { + case .relaxed: return "relaxed" + case .acquiring: return "acquiring" + case .sequentiallyConsistent: return "sequentiallyConsistent" + default: return "AtomicLoadOrdering(\(_rawValue))" + } + } +} + +//------------------------------------------------------------------------------ + +/// Specifies the memory ordering semantics of an atomic store operation. +@frozen +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +public struct AtomicStoreOrdering { + // This struct works like a non-frozen enum whose cases aren't reorderable. + + @usableFromInline + internal var _rawValue: Int + + @_semantics("constant_evaluable") + @inlinable @_transparent // Debug performance + internal init(_rawValue: Int) { + self._rawValue = _rawValue + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicStoreOrdering { + // FIXME: Explain these ordering levels in more detail. + + /// Guarantees the atomicity of the specific operation on which it is applied, + /// but imposes no ordering constraints on any other reads or writes. + @_semantics("constant_evaluable") + @_alwaysEmitIntoClient + @_transparent // Debug performance + public static var relaxed: Self { + Self(_rawValue: 0) + } + + /// A releasing store prevents the effects of previous load and store + /// operations on the current thread from appearing to happen after the effect + /// of the atomic operation itself. + @_semantics("constant_evaluable") + @_alwaysEmitIntoClient + @_transparent // Debug performance + public static var releasing: Self { + Self(_rawValue: 2) + } + + @_semantics("constant_evaluable") + @_alwaysEmitIntoClient + @_transparent // Debug performance + public static var sequentiallyConsistent: Self { + Self(_rawValue: 4) + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicStoreOrdering: Equatable { + @_transparent // Debug performance + public static func ==(left: Self, right: Self) -> Bool { + return left._rawValue == right._rawValue + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicStoreOrdering: Hashable { + @inlinable + public func hash(into hasher: inout Hasher) { + hasher.combine(_rawValue) + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicStoreOrdering: CustomStringConvertible { + public var description: String { + switch self { + case .relaxed: return "relaxed" + case .releasing: return "releasing" + case .sequentiallyConsistent: return "sequentiallyConsistent" + default: return "AtomicStoreOrdering(\(_rawValue))" + } + } +} + +//------------------------------------------------------------------------------ + +/// Specifies the memory ordering semantics of an atomic read-modify-write +/// operation. +@frozen +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +public struct AtomicUpdateOrdering { + // This struct works like a non-frozen enum whose cases aren't reorderable. + + @usableFromInline + internal var _rawValue: Int + + @_semantics("constant_evaluable") + @inlinable @_transparent // Debug performance + internal init(_rawValue: Int) { + self._rawValue = _rawValue + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicUpdateOrdering { + // FIXME: Explain these ordering levels in more detail. + + /// Guarantees the atomicity of the specific operation on which it is applied, + /// but imposes no ordering constraints on any other reads or writes. + @_semantics("constant_evaluable") + @_alwaysEmitIntoClient + @_transparent // Debug performance + public static var relaxed: Self { + Self(_rawValue: 0) + } + + /// An acquiring load prevents the effects of subsequent load and store + /// operations on the current thread from appearing to happen before the + /// effect of the atomic operation itself. + @_semantics("constant_evaluable") + @_alwaysEmitIntoClient + @_transparent // Debug performance + public static var acquiring: Self { + Self(_rawValue: 1) + } + + /// A releasing store prevents the effects of previous load and store + /// operations on the current thread from appearing to happen after the effect + /// of the atomic operation itself. + @_semantics("constant_evaluable") + @_alwaysEmitIntoClient + @_transparent // Debug performance + public static var releasing: Self { + Self(_rawValue: 2) + } + + /// An acquiring-and-releasing operation is a combination of `.acquiring` and + /// `.releasing`; it prevents all neighboring load and store operations on the + /// current thread from appearing to happen in a different order in relation + /// to the atomic operation. + @_semantics("constant_evaluable") + @_alwaysEmitIntoClient + @_transparent // Debug performance + public static var acquiringAndReleasing: Self { + Self(_rawValue: 3) + } + + @_semantics("constant_evaluable") + @_alwaysEmitIntoClient + @_transparent // Debug performance + public static var sequentiallyConsistent: Self { + Self(_rawValue: 4) + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicUpdateOrdering: Equatable { + @_transparent // Debug performance + public static func ==(left: Self, right: Self) -> Bool { + return left._rawValue == right._rawValue + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicUpdateOrdering: Hashable { + @inlinable + public func hash(into hasher: inout Hasher) { + hasher.combine(_rawValue) + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicUpdateOrdering: CustomStringConvertible { + public var description: String { + switch self { + case .relaxed: return "relaxed" + case .acquiring: return "acquiring" + case .releasing: return "releasing" + case .acquiringAndReleasing: return "acquiringAndReleasing" + case .sequentiallyConsistent: return "sequentiallyConsistent" + default: return "AtomicUpdateOrdering(\(_rawValue))" + } + } +} + +@_semantics("atomics.requires_constant_orderings") +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +@_transparent // Debug performance +public func atomicMemoryFence( + ordering: AtomicUpdateOrdering +) { + switch ordering { + case .relaxed: break + case .acquiring: Builtin.fence_acquire() + case .releasing: Builtin.fence_release() + case .acquiringAndReleasing: Builtin.fence_acqrel() + case .sequentiallyConsistent: Builtin.fence_seqcst() + default: Builtin.unreachable() + } +} diff --git a/stdlib/public/Atomics/AtomicOptional.swift b/stdlib/public/Atomics/AtomicOptional.swift new file mode 100644 index 0000000000000..83e255ec6de7a --- /dev/null +++ b/stdlib/public/Atomics/AtomicOptional.swift @@ -0,0 +1,58 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +public protocol _PrimitiveAtomicOptional: _PrimitiveAtomic { + var _isNil: Bool { get } + static var _nilValue: Self { get } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension Optional: AtomicValue +where Wrapped: AtomicValue, Wrapped._AtomicStorage: _PrimitiveAtomicOptional +{ + public typealias _AtomicStorage = Wrapped._AtomicStorage + + @_transparent @_alwaysEmitIntoClient + public static func _prepareAtomicStorage( + for value: __owned Self + ) -> _AtomicStorage { + guard let value = value else { return _AtomicStorage._nilValue } + return Wrapped._prepareAtomicStorage(for: value) + } + + @_transparent @_alwaysEmitIntoClient + public static func _disposeAtomicStorage( + _ storage: inout _AtomicStorage + ) -> Self { + guard !storage._isNil else { return nil } + return Wrapped._disposeAtomicStorage(&storage) + } + + @_transparent @_alwaysEmitIntoClient + public static func _encodeAtomicStorage( + for value: __owned Self + ) -> _AtomicStorage { + guard let value = value else { return _AtomicStorage._nilValue } + return Wrapped._encodeAtomicStorage(for: value) + } + + @_transparent @_alwaysEmitIntoClient + public static func _decodeAtomicStorage( + _ storage: __owned _AtomicStorage + ) -> Self { + guard !storage._isNil else { return nil } + return Wrapped._decodeAtomicStorage(storage) + } +} diff --git a/stdlib/public/Atomics/AtomicPointerConformances.swift.gyb b/stdlib/public/Atomics/AtomicPointerConformances.swift.gyb new file mode 100644 index 0000000000000..3eed105c575d8 --- /dev/null +++ b/stdlib/public/Atomics/AtomicPointerConformances.swift.gyb @@ -0,0 +1,268 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +%{ + atomicTypes = [ + "UnsafeRawPointer", + "UnsafeMutableRawPointer", + "UnsafePointer", + "UnsafeMutablePointer", + "Unmanaged", + ] + + def rmwOrderings(): + return [ + # Swift enum case, llvm name, failure name + ('relaxed', 'monotonic', 'monotonic'), + ('acquiring', 'acquire', 'acquire'), + ('releasing', 'release', 'monotonic'), + ('acquiringAndReleasing', 'acqrel', 'acquire'), + ('sequentiallyConsistent', 'seqcst', 'seqcst'), + ] + + def loadOrderings(): + return [ + # Swift enum case, llvm name + ('relaxed', 'monotonic'), + ('acquiring', 'acquire'), + ('sequentiallyConsistent', 'seqcst'), + ] + + def storeOrderings(): + return [ + # Swift enum case, llvm name + ('relaxed', 'monotonic'), + ('releasing', 'release'), + ('sequentiallyConsistent', 'seqcst'), + ] + + def caseStatementForOrdering(ordering): + return "case .{}".format(ordering) + + # FIXME: Swift should provide intrinsics for arbitrary ordering pairs + def llvmOrders(rmw, load): # See llvm/Support/AtomicOrdering.h + def max(rmw, load): + if load == "acquire": + if rmw == "monotonic": + return "acquire" + if rmw == "release": + return "acqrel" + if load == "seqcst": + return "seqcst" + return rmw + return max(rmw, load) + "_" + load +}% + +import Swift + + +% for swiftType in atomicTypes: +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension ${swiftType} { + @frozen + public struct _AtomicStorage { + @usableFromInline + internal let _word: Builtin.Word + + @_transparent @_alwaysEmitIntoClient + internal init(_word: Builtin.Word) { + self._word = _word + } + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension ${swiftType}: AtomicValue { + @_transparent @_alwaysEmitIntoClient + public static func _prepareAtomicStorage(for value: Self) -> _AtomicStorage { + return _encodeAtomicStorage(for: value) + } + + @_transparent @_alwaysEmitIntoClient + public static func _disposeAtomicStorage( + _ storage: inout _AtomicStorage + ) -> Self { + return _decodeAtomicStorage(storage) + } + + @_transparent @_alwaysEmitIntoClient + public static func _encodeAtomicStorage(for value: Self) -> _AtomicStorage { + % if swiftType == "Unmanaged": + return _AtomicStorage( + _word: Int(bitPattern: value.toOpaque())._builtinWordValue) + % else: + return _AtomicStorage(_word: Int(bitPattern: value)._builtinWordValue) + % end + } + + @_transparent @_alwaysEmitIntoClient + public static func _decodeAtomicStorage(_ storage: _AtomicStorage) -> Self { + % if swiftType == "Unmanaged": + let raw = UnsafeRawPointer(bitPattern: Int(storage._word)) + return Unmanaged.fromOpaque(raw!) + % else: + return ${swiftType}(bitPattern: Int(storage._word))! + % end + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension ${swiftType}._AtomicStorage: _PrimitiveAtomicOptional { + @_transparent @_alwaysEmitIntoClient + public var _isNil: Bool { Int(_word) == 0 } + + @_transparent @_alwaysEmitIntoClient + public static var _nilValue: Self { Self(_word: 0._builtinWordValue) } + + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicLoad( + at pointer: UnsafeMutablePointer, + ordering: AtomicLoadOrdering + ) -> Self { + let word: Builtin.Word + switch ordering { +% for (swiftOrder, llvmOrder) in loadOrderings(): + ${caseStatementForOrdering(swiftOrder)}: + word = Builtin.atomicload_${llvmOrder}_Word(pointer._rawValue) +% end + default: + Builtin.unreachable() + } + return Self(_word: word) + } + + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicStore( + _ desired: Self, + at pointer: UnsafeMutablePointer, + ordering: AtomicStoreOrdering + ) { + switch ordering { +% for (swiftOrder, llvmOrder) in storeOrderings(): + ${caseStatementForOrdering(swiftOrder)}: + Builtin.atomicstore_${llvmOrder}_Word( + pointer._rawValue, + desired._word) +% end + default: + Builtin.unreachable() + } + } + + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicExchange( + _ desired: Self, + at pointer: UnsafeMutablePointer, + ordering: AtomicUpdateOrdering + ) -> Self { + let word: Builtin.Word + switch ordering { +% for (swiftOrder, llvmOrder, _) in rmwOrderings(): + ${caseStatementForOrdering(swiftOrder)}: + word = Builtin.atomicrmw_xchg_${llvmOrder}_Word( + pointer._rawValue, + desired._word) +% end + default: + Builtin.unreachable() + } + return Self(_word: word) + } + + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicCompareExchange( + expected: Self, + desired: Self, + at pointer: UnsafeMutablePointer, + ordering: AtomicUpdateOrdering + ) -> (exchanged: Bool, original: Self) { + let old: Builtin.Word + let won: Builtin.Int1 + switch ordering { +% for (swiftOrder, llvmOrder, failureOrder) in rmwOrderings(): + ${caseStatementForOrdering(swiftOrder)}: + (old, won) = Builtin.cmpxchg_${llvmOrder}_${failureOrder}_Word( + pointer._rawValue, + expected._word, + desired._word) +% end + default: + Builtin.unreachable() + } + return ( + Bool(_builtinBooleanLiteral: won), + Self(_word: old)) + } + + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicCompareExchange( + expected: Self, + desired: Self, + at pointer: UnsafeMutablePointer, + successOrdering: AtomicUpdateOrdering, + failureOrdering: AtomicLoadOrdering + ) -> (exchanged: Bool, original: Self) { + let old: Builtin.Word + let won: Builtin.Int1 + switch (successOrdering, failureOrdering) { +% for (swiftSuccess, llvmSuccess, _) in rmwOrderings(): +% for (swiftFailure, llvmFailure) in loadOrderings(): + case (.${swiftSuccess}, .${swiftFailure}): + (old, won) = Builtin.cmpxchg_${llvmOrders(llvmSuccess, llvmFailure)}_Word( + pointer._rawValue, + expected._word, + desired._word) +% end +% end + default: + Builtin.unreachable() + } + return ( + Bool(_builtinBooleanLiteral: won), + Self(_word: old)) + } + + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public static func _atomicWeakCompareExchange( + expected: Self, + desired: Self, + at pointer: UnsafeMutablePointer, + successOrdering: AtomicUpdateOrdering, + failureOrdering: AtomicLoadOrdering + ) -> (exchanged: Bool, original: Self) { + let old: Builtin.Word + let won: Builtin.Int1 + switch (successOrdering, failureOrdering) { +% for (swiftSuccess, llvmSuccess, _) in rmwOrderings(): +% for (swiftFailure, llvmFailure) in loadOrderings(): + case (.${swiftSuccess}, .${swiftFailure}): + (old, won) = Builtin.cmpxchg_${llvmOrders(llvmSuccess, llvmFailure)}_weak_Word( + pointer._rawValue, + expected._word, + desired._word) +% end +% end + default: + Builtin.unreachable() + } + return ( + Bool(_builtinBooleanLiteral: won), + Self(_word: old)) + } +} +% end diff --git a/stdlib/public/Atomics/AtomicValue.swift b/stdlib/public/Atomics/AtomicValue.swift new file mode 100644 index 0000000000000..ff6ccab17d629 --- /dev/null +++ b/stdlib/public/Atomics/AtomicValue.swift @@ -0,0 +1,128 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +public protocol AtomicValue { + associatedtype _AtomicStorage: _PrimitiveAtomic + + /// Convert `value` to its atomic storage representation. Note that the act of + /// conversion may have side effects (such as retaining a strong reference), + /// so the returned value must be used to initialize exactly one atomic + /// storage location. + /// + /// Each call to `prepareAtomicStorage(for:)` must be paired with call to + /// `disposeAtomicStorage(at:)` to undo these potential side effects. + /// + /// Between initialization and the call to `disposeAtomicStorage`, the + /// memory location must only be accessed through the atomic operations + /// exposed by this type. + /// + /// For example, here is how these methods can be used to create a temporary + /// atomic variable for the duration of a closure call: + /// + /// extension AtomicValue { + /// mutating func withTemporaryAtomicValue( + /// _ body: (UnsafeAtomic) -> Void + /// ) { + /// let storage = + /// UnsafeMutablePointer<_AtomicStorage>.allocate(capacity: 1) + /// storage.initialize(to: Self._prepareAtomicStorage(for: self) + /// defer { + /// Self._disposeAtomicStorage(at: storage) + /// storage.deallocate() + /// } + /// let tmp = UnsafeAtomic(at: storage) + /// body(tmp) + /// self = tmp.load(ordering: .relaxed) + /// } + /// } + /// + /// 42.withTemporaryAtomicValue { atomic in + /// print(atomic.load(ordering: .relaxed) // Prints 42 + /// } + /// + /// - Parameter value: The value to convert. + /// - Returns: The atomic storage representation of `value`. + static func _prepareAtomicStorage(for value: __owned Self) -> _AtomicStorage + + /// Deinitialize atomic storage at the specified memory location. + /// + /// The specified address must have been previously initialized with a value + /// returned from `prepareAtomicStorage(for:)`, and after initialization, it + /// must have only been accessed using the atomic operations provided. + /// + /// - Parameter storage: A storage value previously initialized with a value + /// returned by `prepareAtomicStorage(for:)`. + static func _disposeAtomicStorage(_ storage: inout _AtomicStorage) -> Self + + static func _encodeAtomicStorage(for value: __owned Self) -> _AtomicStorage + static func _decodeAtomicStorage(_ storage: __owned _AtomicStorage) -> Self + +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicValue where + Self: RawRepresentable, + RawValue: AtomicValue, + _AtomicStorage == RawValue._AtomicStorage +{ + @_transparent @_alwaysEmitIntoClient + public static func _prepareAtomicStorage( + for value: __owned Self + ) -> RawValue._AtomicStorage { + return RawValue._prepareAtomicStorage(for: value.rawValue) + } + + @_transparent @_alwaysEmitIntoClient + public static func _disposeAtomicStorage( + _ storage: inout RawValue._AtomicStorage + ) -> Self { + return Self(rawValue: RawValue._disposeAtomicStorage(&storage))! + } + + @_transparent @_alwaysEmitIntoClient + public static func _encodeAtomicStorage( + for value: __owned Self + ) -> RawValue._AtomicStorage { + return RawValue._encodeAtomicStorage(for: value.rawValue) + } + + @_transparent @_alwaysEmitIntoClient + public static func _decodeAtomicStorage( + _ storage: __owned RawValue._AtomicStorage + ) -> Self { + return Self(rawValue: RawValue._decodeAtomicStorage(storage))! + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension AtomicValue where _AtomicStorage == Self { + @_transparent @_alwaysEmitIntoClient + public static func _prepareAtomicStorage(for value: __owned Self) -> Self { + value + } + @_transparent @_alwaysEmitIntoClient + public static func _disposeAtomicStorage(_ storage: inout Self) -> Self { + storage + } + + @_transparent @_alwaysEmitIntoClient + public static func _encodeAtomicStorage(for value: __owned Self) -> Self { + value + } + @_transparent @_alwaysEmitIntoClient + public static func _decodeAtomicStorage(_ storage: __owned Self) -> Self { + storage + } +} diff --git a/stdlib/public/Atomics/CMakeLists.txt b/stdlib/public/Atomics/CMakeLists.txt new file mode 100644 index 0000000000000..dacfde71a1fce --- /dev/null +++ b/stdlib/public/Atomics/CMakeLists.txt @@ -0,0 +1,29 @@ +#===----------------------------------------------------------------------===# +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2020 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +#===----------------------------------------------------------------------===# + +add_swift_target_library(swiftAtomics ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB + AtomicInteger.swift + AtomicMemoryOrderings.swift + AtomicValue.swift + AtomicOptional.swift + PrimitiveAtomic.swift + UnsafeAtomicLazyReference.swift + + GYB_SOURCES + AtomicIntegerConformances.swift.gyb + AtomicPointerConformances.swift.gyb + UnsafeAtomic.swift.gyb + + SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS} "-parse-stdlib" + LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}" + INSTALL_IN_COMPONENT stdlib + MACCATALYST_BUILD_FLAVOR zippered) diff --git a/stdlib/public/Atomics/PrimitiveAtomic.swift b/stdlib/public/Atomics/PrimitiveAtomic.swift new file mode 100644 index 0000000000000..2bf62a6ae3674 --- /dev/null +++ b/stdlib/public/Atomics/PrimitiveAtomic.swift @@ -0,0 +1,177 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +public protocol _PrimitiveAtomic { + /// Atomically loads and returns the value referenced by the given pointer, + /// applying the specified memory ordering. + /// + /// - Parameter pointer: A memory location previously initialized with a value + /// returned by `prepareAtomicStorage(for:)`. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The current value referenced by `pointer`. + @_semantics("atomics.requires_constant_orderings") + static func _atomicLoad( + at pointer: UnsafeMutablePointer, + ordering: AtomicLoadOrdering + ) -> Self + + /// Atomically sets the value referenced by `pointer` to `desired`, + /// applying the specified memory ordering. + /// + /// - Parameter desired: The desired new value. + /// - Parameter pointer: A memory location previously initialized with a value + /// returned by `prepareAtomicStorage(for:)`. + /// - Parameter ordering: The memory ordering to apply on this operation. + @_semantics("atomics.requires_constant_orderings") + static func _atomicStore( + _ desired: __owned Self, + at pointer: UnsafeMutablePointer, + ordering: AtomicStoreOrdering + ) + + /// Atomically sets the value referenced by `pointer` to `desired` and returns + /// the original value, applying the specified memory ordering. + /// + /// - Parameter desired: The desired new value. + /// - Parameter pointer: A memory location previously initialized with a value + /// returned by `prepareAtomicStorage(for:)`. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The original value. + @_semantics("atomics.requires_constant_orderings") + static func _atomicExchange( + _ desired: __owned Self, + at pointer: UnsafeMutablePointer, + ordering: AtomicUpdateOrdering + ) -> Self + + /// Perform an atomic compare and exchange operation on the value referenced + /// by `pointer`, applying the specified memory ordering. + /// + /// This operation performs the following algorithm as a single atomic + /// transaction: + /// + /// ``` + /// atomic(self) { currentValue in + /// let original = currentValue + /// guard original == expected else { return (false, original) } + /// currentValue = desired + /// return (true, original) + /// } + /// ``` + /// + /// This method implements a "strong" compare and exchange operation + /// that does not permit spurious failures. + /// + /// - Parameter expected: The expected current value. + /// - Parameter desired: The desired new value. + /// - Parameter pointer: A memory location previously initialized with a value + /// returned by `prepareAtomicStorage(for:)`. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if + /// the exchange was successful, and `original` is the original value. + @_semantics("atomics.requires_constant_orderings") + static func _atomicCompareExchange( + expected: Self, + desired: __owned Self, + at pointer: UnsafeMutablePointer, + ordering: AtomicUpdateOrdering + ) -> (exchanged: Bool, original: Self) + + /// Perform an atomic compare and exchange operation on the value referenced + /// by `pointer`, applying the specified success/failure memory orderings. + /// + /// This operation performs the following algorithm as a single atomic + /// transaction: + /// + /// ``` + /// atomic(self) { currentValue in + /// let original = currentValue + /// guard original == expected else { return (false, original) } + /// currentValue = desired + /// return (true, original) + /// } + /// ``` + /// + /// The `successOrdering` argument specifies the memory ordering to use when + /// the operation manages to update the current value, while `failureOrdering` + /// will be used when the operation leaves the value intact. + /// + /// This method implements a "strong" compare and exchange operation + /// that does not permit spurious failures. + /// + /// - Parameter expected: The expected current value. + /// - Parameter desired: The desired new value. + /// - Parameter pointer: A memory location previously initialized with a value + /// returned by `prepareSelf(for:)`. + /// - Parameter successOrdering: The memory ordering to apply if this + /// operation performs the exchange. + /// - Parameter failureOrdering: The memory ordering to apply on this + /// operation does not perform the exchange. + /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if + /// the exchange was successful, and `original` is the original value. + @_semantics("atomics.requires_constant_orderings") + static func _atomicCompareExchange( + expected: Self, + desired: __owned Self, + at pointer: UnsafeMutablePointer, + successOrdering: AtomicUpdateOrdering, + failureOrdering: AtomicLoadOrdering + ) -> (exchanged: Bool, original: Self) + + /// Perform an atomic weak compare and exchange operation on the value + /// referenced by `pointer`, applying the specified success/failure memory + /// orderings. This compare-exchange variant is allowed to spuriously fail; it + /// is designed to be called in a loop until it indicates a successful + /// exchange has happened. + /// + /// This operation performs the following algorithm as a single atomic + /// transaction: + /// + /// ``` + /// atomic(self) { currentValue in + /// let original = currentValue + /// guard original == expected else { return (false, original) } + /// currentValue = desired + /// return (true, original) + /// } + /// ``` + /// + /// (In this weak form, transient conditions may cause the `original == + /// expected` check to sometimes return false when the two values are in fact + /// the same.) + /// + /// The `successOrdering` argument specifies the memory ordering to use when the + /// operation manages to update the current value, while `failureOrdering` + /// will be used when the operation leaves the value intact. + /// + /// - Parameter expected: The expected current value. + /// - Parameter desired: The desired new value. + /// - Parameter pointer: A memory location previously initialized with a value + /// returned by `prepareAtomicStorage(for:)`. + /// - Parameter successOrdering: The memory ordering to apply if this + /// operation performs the exchange. + /// - Parameter failureOrdering: The memory ordering to apply on this + /// operation does not perform the exchange. + /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if + /// the exchange was successful, and `original` is the original value. + @_semantics("atomics.requires_constant_orderings") + static func _atomicWeakCompareExchange( + expected: Self, + desired: __owned Self, + at pointer: UnsafeMutablePointer, + successOrdering: AtomicUpdateOrdering, + failureOrdering: AtomicLoadOrdering + ) -> (exchanged: Bool, original: Self) +} diff --git a/stdlib/public/Atomics/UnsafeAtomic.swift.gyb b/stdlib/public/Atomics/UnsafeAtomic.swift.gyb new file mode 100644 index 0000000000000..b110ac7d23d85 --- /dev/null +++ b/stdlib/public/Atomics/UnsafeAtomic.swift.gyb @@ -0,0 +1,382 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +%{ + def lowerFirst(str): + return str[:1].lower() + str[1:] if str else "" + + def argLabel(label): + return label + ": " if label <> "_" else "" + + integerOperations = [ + # Swift name, llvm name, operator, label, doc + ('WrappingIncrement', 'add', '&+', "by", "wrapping add"), + ('WrappingDecrement', 'sub', '&-', "by", "wrapping subtract"), + ('BitwiseAnd', 'and', '&', "with", "bitwise and"), + ('BitwiseOr', 'or', '|', "with", "bitwise or"), + ('BitwiseXor', 'xor', '^', "with", "bitwise xor") + ] + + def argLabel(label): + return label + ": " if label <> "_" else "" + + def defaultValue(name): + return " = 1" if "crement" in name else "" +}% + +import Swift + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +@frozen +public struct UnsafeAtomic { + @frozen + public struct Storage { + @usableFromInline + internal var _value: Value._AtomicStorage + + @inlinable @inline(__always) + internal init(_value: __owned Value._AtomicStorage) { + self._value = _value + } + + @inlinable @inline(__always) + public init(_ value: __owned Value) { + self._value = Value._prepareAtomicStorage(for: value) + } + + @inlinable @inline(__always) + @discardableResult + public mutating func dispose() -> Value { + return Value._disposeAtomicStorage(&_value) + } + } + + @usableFromInline + internal let _storage: UnsafeMutablePointer + + @_transparent // Debug performance + public init( + @_nonEphemeral at pointer: UnsafeMutablePointer + ) { + self._storage = pointer + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension UnsafeAtomic { + @_transparent @_alwaysEmitIntoClient + internal var _ptr: UnsafeMutablePointer { + // As a single-element struct, `Storage` is layout-compatible with its + // stored property. + return UnsafeMutableRawPointer(_storage) + .assumingMemoryBound(to: Value._AtomicStorage.self) + } + + @inlinable + public static func create(initialValue: __owned Value) -> Self { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + ptr.initialize(to: Storage(initialValue)) + return Self(at: ptr) + } + + @discardableResult + @inlinable + public func destroy() -> Value { + let result = _storage.pointee.dispose() + _storage.deallocate() + return result + } +} + + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension UnsafeAtomic { + /// Atomically loads and returns the current value, applying the specified + /// memory ordering. + /// + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The current value. + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public func load( + ordering: AtomicLoadOrdering + ) -> Value { + let encoded = Value._AtomicStorage._atomicLoad(at: _ptr, ordering: ordering) + return Value._decodeAtomicStorage(encoded) + } + + /// Atomically sets the current value to `desired`, applying the specified + /// memory ordering. + /// + /// - Parameter desired: The desired new value. + /// - Parameter ordering: The memory ordering to apply on this operation. + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public func store( + _ desired: __owned Value, + ordering: AtomicStoreOrdering + ) { + Value._AtomicStorage._atomicStore( + Value._encodeAtomicStorage(for: desired), + at: _ptr, + ordering: ordering) + } + + /// Atomically sets the current value to `desired` and returns the original + /// value, applying the specified memory ordering. + /// + /// - Parameter desired: The desired new value. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The original value. + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public func exchange( + _ desired: __owned Value, + ordering: AtomicUpdateOrdering + ) -> Value { + let encoded = Value._AtomicStorage._atomicExchange( + Value._encodeAtomicStorage(for: desired), + at: _ptr, + ordering: ordering) + return Value._decodeAtomicStorage(encoded) + } + + /// Perform an atomic compare and exchange operation on the current value, + /// applying the specified memory ordering. + /// + /// This operation performs the following algorithm as a single atomic + /// transaction: + /// + /// ``` + /// atomic(self) { currentValue in + /// let original = currentValue + /// guard original == expected else { return (false, original) } + /// currentValue = desired + /// return (true, original) + /// } + /// ``` + /// + /// This method implements a "strong" compare and exchange operation + /// that does not permit spurious failures. + /// + /// - Parameter expected: The expected current value. + /// - Parameter desired: The desired new value. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if + /// the exchange was successful, and `original` is the original value. + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public func compareExchange( + expected: Value, + desired: __owned Value, + ordering: AtomicUpdateOrdering + ) -> (exchanged: Bool, original: Value) { + let (exchanged, original) = Value._AtomicStorage._atomicCompareExchange( + expected: Value._encodeAtomicStorage(for: expected), + desired: Value._encodeAtomicStorage(for: desired), + at: _ptr, + ordering: ordering) + return (exchanged, Value._decodeAtomicStorage(original)) + } + + /// Perform an atomic compare and exchange operation on the current value, + /// applying the specified success/failure memory orderings. + /// + /// This operation performs the following algorithm as a single atomic + /// transaction: + /// + /// ``` + /// atomic(self) { currentValue in + /// let original = currentValue + /// guard original == expected else { return (false, original) } + /// currentValue = desired + /// return (true, original) + /// } + /// ``` + /// + /// The `succesOrdering` argument specifies the memory ordering to use when + /// the operation manages to update the current value, while `failureOrdering` + /// will be used when the operation leaves the value intact. + /// + /// This method implements a "strong" compare and exchange operation + /// that does not permit spurious failures. + /// + /// - Parameter expected: The expected current value. + /// - Parameter desired: The desired new value. + /// - Parameter successOrdering: The memory ordering to apply if this + /// operation performs the exchange. + /// - Parameter failureOrdering: The memory ordering to apply on this + /// operation does not perform the exchange. + /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if + /// the exchange was successful, and `original` is the original value. + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public func compareExchange( + expected: Value, + desired: __owned Value, + successOrdering: AtomicUpdateOrdering, + failureOrdering: AtomicLoadOrdering + ) -> (exchanged: Bool, original: Value) { + let (exchanged, original) = Value._AtomicStorage._atomicCompareExchange( + expected: Value._encodeAtomicStorage(for: expected), + desired: Value._encodeAtomicStorage(for: desired), + at: _ptr, + successOrdering: successOrdering, + failureOrdering: failureOrdering) + return (exchanged, Value._decodeAtomicStorage(original)) + } + + /// Perform an atomic weak compare and exchange operation on the current + /// value, applying the specified success/failure memory orderings. This + /// compare-exchange variant is allowed to spuriously fail; it is designed to + /// be called in a loop until it indicates a successful exchange has happened. + /// + /// This operation performs the following algorithm as a single atomic + /// transaction: + /// + /// ``` + /// atomic(self) { currentValue in + /// let original = currentValue + /// guard original == expected else { return (false, original) } + /// currentValue = desired + /// return (true, original) + /// } + /// ``` + /// + /// (In this weak form, transient conditions may cause the `original == + /// expected` check to sometimes return false when the two values are in fact + /// the same.) + /// + /// The `ordering` argument specifies the memory ordering to use when the + /// operation manages to update the current value, while `failureOrdering` + /// will be used when the operation leaves the value intact. + /// + /// - Parameter expected: The expected current value. + /// - Parameter desired: The desired new value. + /// - Parameter successOrdering: The memory ordering to apply if this + /// operation performs the exchange. + /// - Parameter failureOrdering: The memory ordering to apply on this + /// operation does not perform the exchange. + /// - Returns: A tuple `(exchanged, original)`, where `exchanged` is true if + /// the exchange was successful, and `original` is the original value. + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public func weakCompareExchange( + expected: Value, + desired: __owned Value, + successOrdering: AtomicUpdateOrdering, + failureOrdering: AtomicLoadOrdering + ) -> (exchanged: Bool, original: Value) { + let (exchanged, original) = Value._AtomicStorage._atomicWeakCompareExchange( + expected: Value._encodeAtomicStorage(for: expected), + desired: Value._encodeAtomicStorage(for: desired), + at: _ptr, + successOrdering: successOrdering, + failureOrdering: failureOrdering) + return (exchanged, Value._decodeAtomicStorage(original)) + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension UnsafeAtomic where Value: AtomicInteger { + % for (name, _, op, label, doc) in integerOperations: + /// Perform an atomic ${doc} operation and return the original value, with + /// the specified memory ordering. + /// + % if "Wrapping" in name: + /// Note: This operation silently wraps around on overflow, like the + /// `${op}` operator does on `Int` values. + /// + % end + /// - Parameter operand: An integer value. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The original value before the operation. + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public func loadThen${name}( + ${label} operand: Value${defaultValue(name)}, + ordering: AtomicUpdateOrdering + ) -> Value { + let encoded = Value._AtomicStorage._atomicLoadThen${name}( + ${argLabel(label)}Value._encodeAtomicStorage(for: operand), + at: _ptr, + ordering: ordering) + return Value._decodeAtomicStorage(encoded) + } + % end + + % for (name, _, op, label, doc) in integerOperations: + /// Perform an atomic ${doc} operation and return the new value, with + /// the specified memory ordering. + /// + % if "Wrapping" in name: + /// Note: This operation silently wraps around on overflow, like the + /// `${op}` operator does on `Int` values. + /// + % end + /// - Parameter operand: An integer value. + /// - Parameter ordering: The memory ordering to apply on this operation. + /// - Returns: The new value after the operation. + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public func ${lowerFirst(name)}ThenLoad( + ${label} operand: Value${defaultValue(name)}, + ordering: AtomicUpdateOrdering + ) -> Value { + let encoded = Value._AtomicStorage._atomicLoadThen${name}( + ${argLabel(label)}Value._encodeAtomicStorage(for: operand), + at: _ptr, + ordering: ordering) + return Value._decodeAtomicStorage(encoded) ${op} operand + } + % end + + /// Perform an atomic wrapping increment operation with the specified memory + /// ordering. + /// + /// Note: This operation silently wraps around on overflow, like the + /// `&+=` operator does on `Int` values. + /// + /// - Parameter operand: The value to add to the current value. + /// - Parameter ordering: The memory ordering to apply on this operation. + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public func wrappingIncrement( + by operand: Value = 1, + ordering: AtomicUpdateOrdering + ) { + _ = Value._AtomicStorage._atomicLoadThenWrappingIncrement( + by: Value._encodeAtomicStorage(for: operand), + at: _ptr, + ordering: ordering) + } + + /// Perform an atomic wrapping decrement operation with the specified memory + /// ordering. + /// + /// Note: This operation silently wraps around on overflow, like the + /// `&-=` operator does on `Int` values. + /// + /// - Parameter operand: The value to subtract from the current value. + /// - Parameter ordering: The memory ordering to apply on this operation. + @_semantics("atomics.requires_constant_orderings") + @_transparent @_alwaysEmitIntoClient + public func wrappingDecrement( + by operand: Value = 1, + ordering: AtomicUpdateOrdering + ) { + _ = Value._AtomicStorage._atomicLoadThenWrappingDecrement( + by: Value._encodeAtomicStorage(for: operand), + at: _ptr, + ordering: ordering) + } +} diff --git a/stdlib/public/Atomics/UnsafeAtomicLazyReference.swift b/stdlib/public/Atomics/UnsafeAtomicLazyReference.swift new file mode 100644 index 0000000000000..539fbad1113ca --- /dev/null +++ b/stdlib/public/Atomics/UnsafeAtomicLazyReference.swift @@ -0,0 +1,135 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +/// An atomic optional strong reference that can be set (initialized) exactly +/// once, but read many times. +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +@frozen +public struct UnsafeAtomicLazyReference { + public typealias Value = Instance? + + @usableFromInline + internal let _ptr: UnsafeMutablePointer + + @_transparent // Debug performance + public init(@_nonEphemeral at pointer: UnsafeMutablePointer) { + self._ptr = pointer + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension UnsafeAtomicLazyReference { + @frozen + public struct Storage { + @usableFromInline + internal let _word: Builtin.Word + + @inlinable @inline(__always) + internal init(_word: Builtin.Word) { + self._word = _word + } + + @inlinable @inline(__always) + public init() { + // Note: this assumes nil is mapped to 0 + _word = 0._builtinWordValue + } + + @inlinable @inline(__always) + @discardableResult + public mutating func dispose() -> Value { + guard let ptr = UnsafeRawPointer(bitPattern: Int(_word)) else { + return nil + } + self = Self.init() + return Unmanaged.fromOpaque(ptr).takeRetainedValue() + } + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension UnsafeAtomicLazyReference { + @inlinable + public static func create() -> Self { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + ptr.initialize(to: Storage()) + return Self(at: ptr) + } + + @discardableResult + @inlinable + public func destroy() -> Value { + let result = _ptr.pointee.dispose() + _ptr.deallocate() + return result + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension UnsafeAtomicLazyReference { + /// Atomically initializes this reference if its current value is nil, then + /// returns the initialized value. If this reference is already initialized, + /// then `initialize(to:)` discards its supplied argument and returns the + /// current value without updating it. + /// + /// The following example demonstrates how this can be used to implement a + /// thread-safe lazily initialized reference: + /// + /// ``` + /// class Image { + /// var _histogram: UnsafeAtomicLazyReference = ... + /// + /// // This is safe to call concurrently from multiple threads. + /// var atomicLazyHistogram: Histogram { + /// if let histogram = _histogram.load() { return foo } + /// // Note that code here may run concurrently on + /// // multiple threads, but only one of them will get to + /// // succeed setting the reference. + /// let histogram = ... + /// return _histogram.storeIfNilThenLoad(foo) + /// } + /// ``` + /// + /// This operation uses acquiring-and-releasing memory ordering. + public func storeIfNilThenLoad(_ desired: __owned Instance) -> Instance { + let desiredUnmanaged = Unmanaged.passRetained(desired) + let desiredInt = Int(bitPattern: desiredUnmanaged.toOpaque()) + let (current, won) = Builtin.cmpxchg_acqrel_acquire_Word( + _ptr._rawValue, + Storage()._word, + desiredInt._builtinWordValue) + if !Bool(_builtinBooleanLiteral: won) { + // The reference has already been initialized. Balance the retain that + // we performed on `desired`. + desiredUnmanaged.release() + let raw = UnsafeRawPointer(bitPattern: Int(current)).unsafelyUnwrapped + let result = Unmanaged.fromOpaque(raw) + return result.takeUnretainedValue() + } + return desiredUnmanaged.takeUnretainedValue() + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +extension UnsafeAtomicLazyReference { + /// Atomically loads and returns the current value of this reference. + /// + /// The load operation is performed with the memory ordering + /// `AtomicLoadOrdering.acquiring`. + public func load() -> Instance? { + let value = Builtin.atomicload_acquire_Word(_ptr._rawValue) + guard let ptr = UnsafeRawPointer(bitPattern: Int(value)) else { return nil } + return Unmanaged.fromOpaque(ptr).takeUnretainedValue() + } +} diff --git a/stdlib/public/CMakeLists.txt b/stdlib/public/CMakeLists.txt index e8d5da6490b76..87668e6cbf60a 100644 --- a/stdlib/public/CMakeLists.txt +++ b/stdlib/public/CMakeLists.txt @@ -61,6 +61,7 @@ if(SWIFT_BUILD_STDLIB) add_subdirectory(stubs) add_subdirectory(core) add_subdirectory(SwiftOnoneSupport) + add_subdirectory(Atomics) if(SWIFT_ENABLE_EXPERIMENTAL_DIFFERENTIABLE_PROGRAMMING) add_subdirectory(Differentiation) diff --git a/test/Atomics/AtomicRawRepresentable.swift b/test/Atomics/AtomicRawRepresentable.swift new file mode 100644 index 0000000000000..caa3df5c1344e --- /dev/null +++ b/test/Atomics/AtomicRawRepresentable.swift @@ -0,0 +1,81 @@ +// RUN: %target-run-simple-swift +// REQUIRES: executable_test + +import StdlibUnittest +import Atomics + +let suite = TestSuite("AtomicRawRepresentable") +defer { runAllTests() } + +enum State: Int, AtomicValue { + case starting + case running + case stopped +} + + +suite.test("load") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + let v = UnsafeAtomic.create(initialValue: .starting) + defer { v.destroy() } + expectEqual(v.load(ordering: .relaxed), State.starting) +} + +suite.test("store") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + let v = UnsafeAtomic.create(initialValue: .starting) + defer { v.destroy() } + expectEqual(State.starting, v.load(ordering: .relaxed)) + v.store(.running, ordering: .relaxed) + expectEqual(State.running, v.load(ordering: .relaxed)) +} + +suite.test("exchange") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + let v = UnsafeAtomic.create(initialValue: .starting) + defer { v.destroy() } + expectEqual(State.starting, v.load(ordering: .relaxed)) + expectEqual(State.starting, v.exchange(.running, ordering: .relaxed)) + expectEqual(State.running, v.load(ordering: .relaxed)) + expectEqual(State.running, v.exchange(.stopped, ordering: .relaxed)) + expectEqual(State.stopped, v.load(ordering: .relaxed)) +} + +suite.test("compareExchange") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + let v = UnsafeAtomic.create(initialValue: .starting) + defer { v.destroy() } + expectEqual(State.starting, v.load(ordering: .relaxed)) + + var (success, old) = v.compareExchange( + expected: .starting, + desired: .running, + ordering: .relaxed) + expectTrue(success) + expectEqual(State.starting, old) + expectEqual(State.running, v.load(ordering: .relaxed)) + + (success, old) = v.compareExchange( + expected: .starting, + desired: .stopped, + ordering: .relaxed) + expectFalse(success) + expectEqual(.running, old) + expectEqual(State.running, v.load(ordering: .relaxed)) + + (success, old) = v.compareExchange( + expected: .running, + desired: .stopped, + ordering: .relaxed) + expectTrue(success) + expectEqual(State.running, old) + expectEqual(State.stopped, v.load(ordering: .relaxed)) +} diff --git a/test/Atomics/AtomicsFolding.swift.gyb b/test/Atomics/AtomicsFolding.swift.gyb new file mode 100644 index 0000000000000..6d60d8a2372cc --- /dev/null +++ b/test/Atomics/AtomicsFolding.swift.gyb @@ -0,0 +1,537 @@ +// RUN: %empty-directory(%t) +// RUN: %gyb %s > %t/AtomicsFolding.swift +// +// RUN: %target-swift-frontend -parse-as-library -emit-sil -Onone -module-name AtomicsFolding %t/AtomicsFolding.swift > %t/AtomicsFolding-Onone.sil +// RUN: %FileCheck --check-prefixes=SECTIONING,CHECK --input-file=%t/AtomicsFolding-Onone.sil %t/AtomicsFolding.swift +// RUN: %FileCheck --check-prefixes=SECTIONING,EXCLUSIONS --input-file=%t/AtomicsFolding-Onone.sil %t/AtomicsFolding.swift +// +// RUN: %target-swift-frontend -parse-as-library -emit-sil -O -module-name AtomicsFolding %t/AtomicsFolding.swift > %t/AtomicsFolding-O.sil +// RUN: %FileCheck --check-prefixes=SECTIONING,CHECK --input-file=%t/AtomicsFolding-O.sil %t/AtomicsFolding.swift +// RUN: %FileCheck --check-prefixes=SECTIONING,EXCLUSIONS --input-file=%t/AtomicsFolding-O.sil %t/AtomicsFolding.swift +// +// RUN: %target-swift-frontend -parse-as-library -emit-sil -Osize -module-name AtomicsFolding %t/AtomicsFolding.swift > %t/AtomicsFolding-Osize.sil +// RUN: %FileCheck --check-prefixes=SECTIONING,CHECK --input-file=%t/AtomicsFolding-Osize.sil %t/AtomicsFolding.swift +// RUN: %FileCheck --check-prefixes=SECTIONING,EXCLUSIONS --input-file=%t/AtomicsFolding-Osize.sil %t/AtomicsFolding.swift + +// Tests that atomic operations get compiled to the expected builtins, and only +// the expected builtins. (I.e., that the switch statements over the orderings +// get constant folded away.) + +%{ + integer_types = [ + # Label Type Builtin + ("Int", "Int", "Word"), + ("Int64", "Int64", "Int64"), + ("Int32", "Int32", "Int32"), + ("Int16", "Int16", "Int16"), + ("Int8", "Int8", "Int8"), + ("UInt", "UInt", "Word"), + ("UInt64", "UInt64", "Int64"), + ("UInt32", "UInt32", "Int32"), + ("UInt16", "UInt16", "Int16"), + ("UInt8", "UInt8", "Int8"), + ] + + pointer_types = [ + # Label Type Builtin + ("UnsafeRawPointer", "UnsafeRawPointer", "Word"), + ("UnsafeMutableRawPointer", "UnsafeMutableRawPointer", "Word"), + ("UnsafePointer", "UnsafePointer", "Word"), + ("UnsafeMutablePointer", "UnsafeMutablePointer", "Word"), + ("Unmanaged", "Unmanaged", "Word"), + ("OptionalUnsafeRawPointer", "Optional", "Word"), + ("OptionalUnsafeMutableRawPointer", "Optional", "Word"), + ("OptionalUnsafePointer", "Optional>", "Word"), + ("OptionalUnsafeMutablePointer", "Optional>", "Word"), + ("OptionalUnmanaged", "Optional>", "Word"), + ] + + all_types = integer_types + pointer_types + + integer_operations = [ + # API Name LLVM Op Arg Label + ('loadThenWrappingIncrement', 'add', "by"), + ('loadThenWrappingDecrement', 'sub', "by"), + ('loadThenBitwiseAnd', 'and', "with"), + ('loadThenBitwiseOr', 'or', "with"), + ('loadThenBitwiseXor', 'xor', "with"), + ('wrappingIncrementThenLoad', 'add', "by"), + ('wrappingDecrementThenLoad', 'sub', "by"), + ('bitwiseAndThenLoad', 'and', "with"), + ('bitwiseOrThenLoad', 'or', "with"), + ('bitwiseXorThenLoad', 'xor', "with"), + ] +}% + +import Atomics + +public struct Foo {} +public class Bar {} + +// ----------------------------------------------------------------------------- + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +public struct AtomicFoldingTests { + % for (label, type, builtin) in all_types: + // SECTIONING-LABEL: // AtomicFoldingTests.load_${label}_relaxed(_:) + public func load_${label}_relaxed( + _ value: UnsafeAtomic<${type}> + ) -> ${type} { + // CHECK: builtin "atomicload_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicload_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicload_seqcst_${builtin}" + return value.load(ordering: .relaxed) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.load_${label}_acquiring(_:) + public func load_${label}_acquiring( + _ value: UnsafeAtomic<${type}> + ) -> ${type} { + // EXCLUSIONS-NOT: builtin "atomicload_monotonic_${builtin}" + // CHECK: builtin "atomicload_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicload_seqcst_${builtin}" + return value.load(ordering: .acquiring) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.load_${label}_sequential(_:) + public func load_${label}_sequential( + _ value: UnsafeAtomic<${type}> + ) -> ${type} { + // EXCLUSIONS-NOT: builtin "atomicload_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicload_acquire_${builtin}" + // CHECK: builtin "atomicload_seqcst_${builtin}" + return value.load(ordering: .sequentiallyConsistent) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + %end + // --------------------------------------------------------------------------- + + % for (label, type, builtin) in all_types: + // SECTIONING-LABEL: // AtomicFoldingTests.store_${label}_relaxed(_:_:) + public func store_${label}_relaxed( + _ value: UnsafeAtomic<${type}>, + _ desired: ${type} + ) { + // CHECK: builtin "atomicstore_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicstore_release_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicstore_seqcst_${builtin}" + value.store(desired, ordering: .relaxed) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.store_${label}_releasing(_:_:) + public func store_${label}_releasing( + _ value: UnsafeAtomic<${type}>, + _ desired: ${type} + ) { + // EXCLUSIONS-NOT: builtin "atomicstore_monotonic_${builtin}" + // CHECK: builtin "atomicstore_release_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicstore_seqcst_${builtin}" + value.store(desired, ordering: .releasing) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.store_${label}_sequential(_:_:) + public func store_${label}_sequential( + _ value: UnsafeAtomic<${type}>, + _ desired: ${type} + ) { + // EXCLUSIONS-NOT: builtin "atomicstore_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicstore_release_${builtin}" + // CHECK: builtin "atomicstore_seqcst_${builtin}" + value.store(desired, ordering: .sequentiallyConsistent) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + %end + // --------------------------------------------------------------------------- + + % for (label, type, builtin) in all_types: + // SECTIONING-LABEL: // AtomicFoldingTests.exchange_${label}_relaxed(_:_:) + public func exchange_${label}_relaxed( + _ value: UnsafeAtomic<${type}>, + _ desired: ${type} + ) -> ${type} { + // CHECK: builtin "atomicrmw_xchg_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_release_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_acqrel_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_seqcst_${builtin}" + return value.exchange(desired, ordering: .relaxed) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.exchange_${label}_acquiring(_:_:) + public func exchange_${label}_acquiring( + _ value: UnsafeAtomic<${type}>, + _ desired: ${type} + ) -> ${type} { + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_monotonic_${builtin}" + // CHECK: builtin "atomicrmw_xchg_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_release_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_acqrel_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_seqcst_${builtin}" + return value.exchange(desired, ordering: .acquiring) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.exchange_${label}_releasing(_:_:) + public func exchange_${label}_releasing( + _ value: UnsafeAtomic<${type}>, + _ desired: ${type} + ) -> ${type} { + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_acquire_${builtin}" + // CHECK: builtin "atomicrmw_xchg_release_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_acqrel_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_seqcst_${builtin}" + return value.exchange(desired, ordering: .releasing) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.exchange_${label}_acquiringAndReleasing(_:_:) + public func exchange_${label}_acquiringAndReleasing( + _ value: UnsafeAtomic<${type}>, + _ desired: ${type} + ) -> ${type} { + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_release_${builtin}" + // CHECK: builtin "atomicrmw_xchg_acqrel_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_seqcst_${builtin}" + return value.exchange(desired, ordering: .acquiringAndReleasing) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.exchange_${label}_sequential(_:_:) + public func exchange_${label}_sequential( + _ value: UnsafeAtomic<${type}>, + _ desired: ${type} + ) -> ${type} { + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_release_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_xchg_acqrel_${builtin}" + // CHECK: builtin "atomicrmw_xchg_seqcst_${builtin}" + return value.exchange(desired, ordering: .sequentiallyConsistent) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + %end + // --------------------------------------------------------------------------- + + % for (label, type, builtin) in all_types: + // SECTIONING-LABEL: // AtomicFoldingTests.compareExchange_${label}_relaxed(_:_:_:) + public func compareExchange_${label}_relaxed( + _ value: UnsafeAtomic<${type}>, + _ expected: ${type}, + _ desired: ${type} + ) -> (exchanged: Bool, original: ${type}) { + // CHECK: builtin "cmpxchg_monotonic_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acquire_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_release_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acqrel_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_seqcst_{{[a-z0-9_]*}}_${builtin}" + return value.compareExchange( + expected: expected, + desired: desired, + ordering: .relaxed) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.compareExchange_${label}_acquiring(_:_:_:) + public func compareExchange_${label}_acquiring( + _ value: UnsafeAtomic<${type}>, + _ expected: ${type}, + _ desired: ${type} + ) -> (exchanged: Bool, original: ${type}) { + // EXCLUSIONS-NOT: builtin "cmpxchg_monotonic_{{[a-z0-9_]*}}_${builtin}" + // CHECK: builtin "cmpxchg_acquire_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_release_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acqrel_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_seqcst_{{[a-z0-9_]*}}_${builtin}" + return value.compareExchange( + expected: expected, + desired: desired, + ordering: .acquiring) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.compareExchange_${label}_releasing(_:_:_:) + public func compareExchange_${label}_releasing( + _ value: UnsafeAtomic<${type}>, + _ expected: ${type}, + _ desired: ${type} + ) -> (exchanged: Bool, original: ${type}) { + // EXCLUSIONS-NOT: builtin "cmpxchg_monotonic_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acquire_{{[a-z0-9_]*}}_${builtin}" + // CHECK: builtin "cmpxchg_release_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acqrel_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_seqcst_{{[a-z0-9_]*}}_${builtin}" + return value.compareExchange( + expected: expected, + desired: desired, + ordering: .releasing) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.compareExchange_${label}_acquiringAndReleasing(_:_:_:) + public func compareExchange_${label}_acquiringAndReleasing( + _ value: UnsafeAtomic<${type}>, + _ expected: ${type}, + _ desired: ${type} + ) -> (exchanged: Bool, original: ${type}) { + // EXCLUSIONS-NOT: builtin "cmpxchg_monotonic_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acquire_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_release_{{[a-z0-9_]*}}_${builtin}" + // CHECK: builtin "cmpxchg_acqrel_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_seqcst_{{[a-z0-9_]*}}_${builtin}" + return value.compareExchange( + expected: expected, + desired: desired, + ordering: .acquiringAndReleasing) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.compareExchange_${label}_sequential(_:_:_:) + public func compareExchange_${label}_sequential( + _ value: UnsafeAtomic<${type}>, + _ expected: ${type}, + _ desired: ${type} + ) -> (exchanged: Bool, original: ${type}) { + // EXCLUSIONS-NOT: builtin "cmpxchg_monotonic_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acquire_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_release_{{[a-z0-9_]*}}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acqrel_{{[a-z0-9_]*}}_${builtin}" + // CHECK: builtin "cmpxchg_seqcst_seqcst_${builtin}" + return value.compareExchange( + expected: expected, + desired: desired, + ordering: .sequentiallyConsistent) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + % end + + % for (operation, modifier) in [("compareExchange", ""), ("weakCompareExchange", "_weak")]: + // --------------------------------------------------------------------------- + // FIXME: There are 15 valid combinations of success/failure orderings; + // here we're only testing 6 of them. + + % for (label, type, builtin) in all_types: + // SECTIONING-LABEL: // AtomicFoldingTests.compareExchange2_${label}_relaxed_relaxed${modifier}(_:_:_:) + public func compareExchange2_${label}_relaxed_relaxed${modifier}( + _ value: UnsafeAtomic<${type}>, + _ expected: ${type}, + _ desired: ${type} + ) -> (exchanged: Bool, original: ${type}) { + // CHECK: builtin "cmpxchg_monotonic_monotonic${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_monotonic_{{[^m][a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acquire_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_release_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acqrel_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_seqcst_{{[a-z0-9_]*}}${modifier}_${builtin}" + return value.${operation}( + expected: expected, + desired: desired, + successOrdering: .relaxed, + failureOrdering: .relaxed) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.compareExchange2_${label}_acquiring_relaxed${modifier}(_:_:_:) + public func compareExchange2_${label}_acquiring_relaxed${modifier}( + _ value: UnsafeAtomic<${type}>, + _ expected: ${type}, + _ desired: ${type} + ) -> (exchanged: Bool, original: ${type}) { + // CHECK: builtin "cmpxchg_acquire_monotonic${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_monotonic_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acquire_{{[^m][a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_release_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acqrel_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_seqcst_{{[a-z0-9_]*}}${modifier}_${builtin}" + return value.${operation}( + expected: expected, + desired: desired, + successOrdering: .acquiring, + failureOrdering: .relaxed) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.compareExchange2_${label}_acquiring_acquiring${modifier}(_:_:_:) + public func compareExchange2_${label}_acquiring_acquiring${modifier}( + _ value: UnsafeAtomic<${type}>, + _ expected: ${type}, + _ desired: ${type} + ) -> (exchanged: Bool, original: ${type}) { + // CHECK: builtin "cmpxchg_acquire_acquire${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_monotonic_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acquire_{{[^a][a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_release_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acqrel_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_seqcst_{{[a-z0-9_]*}}${modifier}_${builtin}" + return value.${operation}( + expected: expected, + desired: desired, + successOrdering: .acquiring, + failureOrdering: .acquiring) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.compareExchange2_${label}_sequential_acquiring${modifier}(_:_:_:) + public func compareExchange2_${label}_sequential_acquiring${modifier}( + _ value: UnsafeAtomic<${type}>, + _ expected: ${type}, + _ desired: ${type} + ) -> (exchanged: Bool, original: ${type}) { + // CHECK: builtin "cmpxchg_seqcst_acquire${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_monotonic_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acquire_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_release_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acqrel_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_seqcst_{{[^a][a-z0-9_]*}}${modifier}_${builtin}" + return value.${operation}( + expected: expected, + desired: desired, + successOrdering: .sequentiallyConsistent, + failureOrdering: .acquiring) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + + // SECTIONING-LABEL: // AtomicFoldingTests.compareExchange2_${label}_releasing_acquiring${modifier}(_:_:_:) + public func compareExchange2_${label}_releasing_acquiring${modifier}( + _ value: UnsafeAtomic<${type}>, + _ expected: ${type}, + _ desired: ${type} + ) -> (exchanged: Bool, original: ${type}) { + // FIXME: This should be checking for "cmpxchg_release_acquire", but we don't + // support arbitrary failure orderings yet, so internally we promote success + // orderings to be at least as strong as the failure ordering. + + // CHECK: builtin "cmpxchg_acqrel_acquire${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_monotonic_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acquire_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_release_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acqrel_{{[^a][a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_seqcst_{{[a-z0-9_]*}}${modifier}_${builtin}" + return value.${operation}( + expected: expected, + desired: desired, + successOrdering: .releasing, + failureOrdering: .acquiring) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.compareExchange2_${label}_relaxed_sequential${modifier}(_:_:_:) + public func compareExchange2_${label}_relaxed_sequential${modifier}( + _ value: UnsafeAtomic<${type}>, + _ expected: ${type}, + _ desired: ${type} + ) -> (exchanged: Bool, original: ${type}) { + // FIXME: This should be checking for "cmpxchg_monotonic_seqcst", but we don't + // support arbitrary failure orderings yet, so internally we promote success + // orderings to be at least as strong as the failure ordering. + + // CHECK: builtin "cmpxchg_seqcst_seqcst${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_monotonic_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acquire_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_release_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_acqrel_{{[a-z0-9_]*}}${modifier}_${builtin}" + // EXCLUSIONS-NOT: builtin "cmpxchg_seqcst_{{[^s][a-z0-9_]*}}${modifier}_${builtin}" + return value.${operation}( + expected: expected, + desired: desired, + successOrdering: .relaxed, + failureOrdering: .sequentiallyConsistent) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + % end + % end + // --------------------------------------------------------------------------- + + % for (label, type, builtin) in integer_types: + % for (operation, llvmOp, argLabel) in integer_operations: + // SECTIONING-LABEL: // AtomicFoldingTests.${operation}_${label}_relaxed(_:_:) + public func ${operation}_${label}_relaxed( + _ value: UnsafeAtomic<${type}>, + _ operand: ${type} + ) -> ${type} { + // CHECK: builtin "atomicrmw_${llvmOp}_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_release_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_acqrel_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_seqcst_${builtin}" + return value.${operation}(${argLabel}: operand, ordering: .relaxed) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.${operation}_${label}_acquiring(_:_:) + public func ${operation}_${label}_acquiring( + _ value: UnsafeAtomic<${type}>, + _ operand: ${type} + ) -> ${type} { + // CHECK: builtin "atomicrmw_${llvmOp}_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_release_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_acqrel_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_seqcst_${builtin}" + return value.${operation}(${argLabel}: operand, ordering: .acquiring) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.${operation}_${label}_releasing(_:_:) + public func ${operation}_${label}_releasing( + _ value: UnsafeAtomic<${type}>, + _ operand: ${type} + ) -> ${type} { + // CHECK: builtin "atomicrmw_${llvmOp}_release_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_acqrel_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_seqcst_${builtin}" + return value.${operation}(${argLabel}: operand, ordering: .releasing) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.${operation}_${label}_acquiringAndReleasing(_:_:) + public func ${operation}_${label}_acquiringAndReleasing( + _ value: UnsafeAtomic<${type}>, + _ operand: ${type} + ) -> ${type} { + // CHECK: builtin "atomicrmw_${llvmOp}_acqrel_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_release_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_seqcst_${builtin}" + return value.${operation}( + ${argLabel}: operand, + ordering: .acquiringAndReleasing) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + // SECTIONING-LABEL: // AtomicFoldingTests.${operation}_${label}_sequential(_:_:) + public func ${operation}_${label}_sequential( + _ value: UnsafeAtomic<${type}>, + _ operand: ${type} + ) -> ${type} { + // CHECK: builtin "atomicrmw_${llvmOp}_seqcst_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_monotonic_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_acquire_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_release_${builtin}" + // EXCLUSIONS-NOT: builtin "atomicrmw_${llvmOp}_acqrel_${builtin}" + return value.${operation}( + ${argLabel}: operand, + ordering: .sequentiallyConsistent) + } + // SECTIONING-LABEL: } // end sil function '$s14AtomicsFolding + + % end + % end + // --------------------------------------------------------------------------- +} diff --git a/test/Atomics/LockFreeSingleConsumerStack.swift b/test/Atomics/LockFreeSingleConsumerStack.swift new file mode 100644 index 0000000000000..c59018b9062ba --- /dev/null +++ b/test/Atomics/LockFreeSingleConsumerStack.swift @@ -0,0 +1,152 @@ +// RUN: %empty-directory(%t) +// RUN: %target-build-swift %s %import-libdispatch -o %t/LockFreeSingleConsumerStack +// RUN: %target-codesign %t/LockFreeSingleConsumerStack +// RUN: %target-run %t/LockFreeSingleConsumerStack +// REQUIRES: executable_test +// REQUIRES: libdispatch + +// FIXME Run with TSAN + +import StdlibUnittest +import Atomics +import Dispatch + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +class LockFreeSingleConsumerStack { + struct Node { + let value: Element + var next: UnsafeMutablePointer? + } + typealias NodePtr = UnsafeMutablePointer + + private var _last = UnsafeAtomic.create(initialValue: nil) + private var _consumerCount = UnsafeAtomic.create(initialValue: 0) + + deinit { + // Discard remaining nodes + while let _ = pop() {} + _last.destroy() + _consumerCount.destroy() + } + + // Push the given element to the top of the stack. + // It is okay to concurrently call this in an arbitrary number of threads. + func push(_ value: Element) { + let new = NodePtr.allocate(capacity: 1) + new.initialize(to: Node(value: value, next: nil)) + + var done = false + var current = _last.load(ordering: .relaxed) + while !done { + new.pointee.next = current + (done, current) = _last.compareExchange( + expected: current, + desired: new, + ordering: .releasing) + } + } + + // Pop and return the topmost element from the stack. + // This method does not support multiple overlapping concurrent calls. + func pop() -> Element? { + precondition( + _consumerCount.loadThenWrappingIncrement(ordering: .acquiring) == 0, + "Multiple consumers detected") + defer { _consumerCount.wrappingDecrement(ordering: .releasing) } + var done = false + var current = _last.load(ordering: .acquiring) + while let c = current { + (done, current) = _last.compareExchange( + expected: c, + desired: c.pointee.next, + ordering: .acquiring) + if done { + let result = c.move() + c.deallocate() + return result.value + } + } + return nil + } +} + +let suite = TestSuite("LockFreeSingleConsumerStack") +defer { runAllTests() } + +suite.test("Basics") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + let stack = LockFreeSingleConsumerStack() + expectNil(stack.pop()) + stack.push(0) + expectEqual(0, stack.pop()) + + stack.push(1) + stack.push(2) + stack.push(3) + stack.push(4) + expectEqual(4, stack.pop()) + expectEqual(3, stack.pop()) + expectEqual(2, stack.pop()) + expectEqual(1, stack.pop()) + expectNil(stack.pop()) +} + +suite.test("ConcurrentPushes") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + let stack = LockFreeSingleConsumerStack<(thread: Int, value: Int)>() + + let numThreads = 100 + let numValues = 10_000 + DispatchQueue.concurrentPerform(iterations: numThreads) { thread in + for value in 1 ... numValues { + stack.push((thread: thread, value: value)) + } + } + + var expected: [Int] = Array(repeating: numValues, count: numThreads) + while let (thread, value) = stack.pop() { + expectEqual(expected[thread], value) + expected[thread] -= 1 + } + expectEqual(Array(repeating: 0, count: numThreads), expected) +} + +suite.test("ConcurrentPushesAndPops") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + let stack = LockFreeSingleConsumerStack<(thread: Int, value: Int)>() + + let numThreads = 100 + let numValues = 10_000 + + var perThreadSums: [Int] = Array(repeating: 0, count: numThreads) + let consumerQueue = DispatchQueue(label: "org.swift.background") + consumerQueue.async { + var count = 0 + while count < numThreads * numValues { + // Note: busy wait + if let (thread, value) = stack.pop() { + perThreadSums[thread] += value + count += 1 + } + } + } + + DispatchQueue.concurrentPerform(iterations: numThreads + 1) { thread in + if thread < numThreads { + // Producers + for value in 0 ..< numValues { + stack.push((thread: thread, value: value)) + } + } + } + + consumerQueue.sync { + expectEqual(Array(repeating: numValues * (numValues - 1) / 2, count: numThreads), perThreadSums) + } +} diff --git a/test/Atomics/UnsafeAtomicInitializers.swift.gyb b/test/Atomics/UnsafeAtomicInitializers.swift.gyb new file mode 100644 index 0000000000000..46d4071b0b1ca --- /dev/null +++ b/test/Atomics/UnsafeAtomicInitializers.swift.gyb @@ -0,0 +1,161 @@ +// RUN: %empty-directory(%t) +// RUN: %gyb -DCMAKE_SIZEOF_VOID_P=%target-ptrsize %s -o %t/UnsafeAtomicInitializers.swift +// RUN: %line-directive %t/UnsafeAtomicInitializers.swift -- %target-swift-frontend -typecheck -verify %t/UnsafeAtomicInitializers.swift + +%{ + types = [ + # id type initial + ("Int", "Int", "0"), + ("Int64", "Int64", "0"), + ("Int32", "Int32", "0"), + ("Int16", "Int16", "0"), + ("Int8", "Int8", "0"), + ("UInt", "UInt", "0"), + ("UInt64", "UInt64", "0"), + ("UInt32", "UInt32", "0"), + ("UInt16", "UInt16", "0"), + ("UInt8", "UInt8", "0"), + + # id type initial + ("URP", "UnsafeRawPointer", "UnsafeRawPointer(UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8))"), + ("UP", "UnsafePointer", "UnsafePointer(UnsafeMutablePointer.allocate(capacity: 1))"), + ("UMRP", "UnsafeMutableRawPointer", ".allocate(byteCount: 8, alignment: 8)"), + ("UMP", "UnsafeMutablePointer", ".allocate(capacity: 1)"), + ("Unmanaged", "Unmanaged", "Unmanaged.passRetained(Foo())"), + + # id type initial + ("URPOpt", "UnsafeRawPointer?", "nil"), + ("UPOpt", "UnsafePointer?", "nil"), + ("UMRPOpt", "UnsafeMutableRawPointer?", "nil"), + ("UMPOpt", "UnsafeMutablePointer?", "nil"), + ("UnmanagedOpt", "Unmanaged?", "nil"), + ] +}% + +import Atomics + +class Foo { + var value = 0 +} +struct Bar { + var value = 0 +} + +% for (id, type, initial) in types: +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +func test_${id}() -> UnsafeAtomic<${type}> { + var storage = UnsafeAtomic<${type}>.Storage(${initial}) + let atomic = UnsafeAtomic<${type}>(at: &storage) // expected-warning {{inout expression creates a temporary pointer, but argument 'at' should be a pointer that outlives the call to 'init(at:)'}} + // expected-note@-1 {{implicit argument conversion from 'UnsafeAtomic<${type}>.Storage' to 'UnsafeMutablePointer.Storage>' produces a pointer valid only for the duration of the call to 'init(at:)'}} + // expected-note@-2 {{use 'withUnsafeMutablePointer' in order to explicitly convert argument to pointer valid for a defined scope}} + return atomic +} +% end + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +func test_UnsafeAtomicLazyReference() -> UnsafeAtomicLazyReference { + var value = UnsafeAtomicLazyReference.Storage() + let atomic = UnsafeAtomicLazyReference(at: &value) // expected-warning {{inout expression creates a temporary pointer, but argument 'at' should be a pointer that outlives the call to 'init(at:)'}} + // expected-note@-1 {{implicit argument conversion from 'UnsafeAtomicLazyReference.Storage' to 'UnsafeMutablePointer.Storage>' produces a pointer valid only for the duration of the call to 'init(at:)'}} + // expected-note@-2 {{use 'withUnsafeMutablePointer' in order to explicitly convert argument to pointer valid for a defined scope}} + return atomic +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +class BrokenAtomicCounter { // THIS IS BROKEN; DO NOT USE + private var _storage = UnsafeAtomic.Storage(0) + private var _value: UnsafeAtomic? + + init() { + // This escapes the ephemeral pointer generated by the inout expression, + // so it leads to undefined behavior when the pointer gets dereferenced + // in the atomic operations below. DO NOT DO THIS. + _value = UnsafeAtomic(at: &_storage) // expected-warning {{inout expression creates a temporary pointer, but argument 'at' should be a pointer that outlives the call to 'init(at:)'}} + // expected-note@-1 {{implicit argument conversion from 'UnsafeAtomic.Storage' to 'UnsafeMutablePointer.Storage>' produces a pointer valid only for the duration of the call to 'init(at:)'}} + // expected-note@-2 {{use 'withUnsafeMutablePointer' in order to explicitly convert argument to pointer valid for a defined scope}} + } + + func increment() { + _value!.wrappingIncrement(by: 1, ordering: .relaxed) + } + + func get() -> Int { + _value!.load(ordering: .relaxed) + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +struct AtomicCounter { + typealias Value = Int + typealias Header = UnsafeAtomic.Storage + + class Buffer: ManagedBuffer { + deinit { + withUnsafeMutablePointerToHeader { header in + _ = header.pointee.dispose() + } + } + } + + let buffer: Buffer + + init() { + buffer = Buffer.create(minimumCapacity: 0) { _ in + Header(0) + } as! Buffer + } + + private func _withAtomicPointer( + _ body: (UnsafeAtomic) throws -> R + ) rethrows -> R { + try buffer.withUnsafeMutablePointerToHeader { header in + try body(UnsafeAtomic(at: header)) + } + } + + func increment() { + _withAtomicPointer { $0.wrappingIncrement(ordering: .relaxed) } + } + + func load() -> Int { + _withAtomicPointer { $0.load(ordering: .relaxed) } + } +} + +@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) +struct AtomicUnmanagedRef { + typealias Value = Unmanaged? + typealias Header = UnsafeAtomic.Storage + + class Buffer: ManagedBuffer { + deinit { + withUnsafeMutablePointerToHeader { header in + _ = header.pointee.dispose() + } + } + } + + let buffer: Buffer + + init() { + buffer = Buffer.create(minimumCapacity: 0) { _ in + Header(nil) + } as! Buffer + } + + private func _withAtomicPointer( + _ body: (UnsafeAtomic) throws -> R + ) rethrows -> R { + try buffer.withUnsafeMutablePointerToHeader { header in + try body(UnsafeAtomic(at: header)) + } + } + + func store(_ desired: Value) { + _withAtomicPointer { $0.store(desired, ordering: .sequentiallyConsistent) } + } + + func load() -> Value { + _withAtomicPointer { $0.load(ordering: .sequentiallyConsistent) } + } +} diff --git a/test/Atomics/UnsafeAtomicIntegers.swift.gyb b/test/Atomics/UnsafeAtomicIntegers.swift.gyb new file mode 100644 index 0000000000000..d42bc7c22e288 --- /dev/null +++ b/test/Atomics/UnsafeAtomicIntegers.swift.gyb @@ -0,0 +1,258 @@ +// RUN: %empty-directory(%t) +// RUN: %gyb -DCMAKE_SIZEOF_VOID_P=%target-ptrsize %s -o %t/UnsafeAtomicIntegers.swift +// RUN: %line-directive %t/UnsafeAtomicIntegers.swift -- %target-build-swift %t/UnsafeAtomicIntegers.swift -o %t/a.out +// RUN: %target-codesign %t/a.out +// RUN: %line-directive %t/UnsafeAtomicIntegers.swift -- %target-run %t/a.out +// REQUIRES: executable_test + +%{ + atomicTypes = [ + "Int", "UInt", + "Int64", "UInt64", + "Int32", "UInt32", + "Int16", "UInt16", + "Int8", "UInt8", + ] + + rmwOrderings = [ + 'relaxed', + 'acquiring', + 'releasing', + 'acquiringAndReleasing', + 'sequentiallyConsistent', + ] + + loadOrderings = [ + 'relaxed', + 'acquiring', + 'sequentiallyConsistent', + ] + + storeOrderings = [ + 'relaxed', + 'releasing', + 'sequentiallyConsistent', + ] + + def lowerFirst(str): + return str[:1].lower() + str[1:] if str else "" + + def argLabel(label): + return label + ": " if label <> "_" else "" + + integerOperations = [ + # Swift name, llvm name, operator, label, doc + ('WrappingIncrement', 'add', '&+', "by", "wrapping add"), + ('WrappingDecrement', 'sub', '&-', "by", "wrapping subtract"), + ('BitwiseAnd', 'and', '&', "with", "bitwise and"), + ('BitwiseOr', 'or', '|', "with", "bitwise or"), + ('BitwiseXor', 'xor', '^', "with", "bitwise xor") + ] + + def argLabel(label): + return label + ": " if label <> "_" else "" +}% + +import StdlibUnittest +import Atomics + +defer { runAllTests() } +let suite = TestSuite("UnsafeAtomicIntegers") + +% for type in atomicTypes: + +suite.test("UnsafeAtomic<${type}>.create-destroy") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + + let v = UnsafeAtomic<${type}>.create(initialValue: 0) + defer { v.destroy() } + expectEqual(v.load(ordering: .relaxed), 0) + + let w = UnsafeAtomic<${type}>.create(initialValue: 42) + defer { w.destroy() } + expectEqual(w.load(ordering: .relaxed), 42) +} + +% for order in loadOrderings: +suite.test("UnsafeAtomic<${type}>.load.${order}") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + + let v = UnsafeAtomic<${type}>.create(initialValue: 0) + defer { v.destroy() } + expectEqual(v.load(ordering: .${order}), 0) + + let w = UnsafeAtomic<${type}>.create(initialValue: 23) + defer { w.destroy() } + expectEqual(w.load(ordering: .${order}), 23) +} +% end + +% for order in storeOrderings: +suite.test("UnsafeAtomic<${type}>.store.${order}") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + + let v = UnsafeAtomic<${type}>.create(initialValue: 0) + defer { v.destroy() } + + v.store(23, ordering: .${order}) + expectEqual(v.load(ordering: .relaxed), 23) + + v.store(42, ordering: .${order}) + expectEqual(v.load(ordering: .relaxed), 42) +} +% end + +% for order in rmwOrderings: +suite.test("UnsafeAtomic<${type}>.exchange.${order}") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + + let v = UnsafeAtomic<${type}>.create(initialValue: 0) + defer { v.destroy() } + + expectEqual(v.exchange(42, ordering: .${order}), 0) + expectEqual(v.load(ordering: .relaxed), 42) + + expectEqual(v.exchange(23, ordering: .${order}), 42) + expectEqual(v.load(ordering: .relaxed), 23) +} +% end + +% for order in rmwOrderings: +suite.test("UnsafeAtomic<${type}>.compareExchange.${order}") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + + let v = UnsafeAtomic<${type}>.create(initialValue: 0) + defer { v.destroy() } + + var (exchanged, original) = v.compareExchange( + expected: 0, + desired: 1, + ordering: .${order}) + expectTrue(exchanged) + expectEqual(original, 0) + expectEqual(v.load(ordering: .relaxed), 1) + + (exchanged, original) = v.compareExchange( + expected: 0, + desired: 2, + ordering: .${order}) + expectFalse(exchanged) + expectEqual(original, 1) + expectEqual(v.load(ordering: .relaxed), 1) + + (exchanged, original) = v.compareExchange( + expected: 1, + desired: 2, + ordering: .${order}) + expectTrue(exchanged) + expectEqual(original, 1) + expectEqual(v.load(ordering: .relaxed), 2) +} +% end + +% for operation in ["compareExchange", "weakCompareExchange"]: +% for successorder in rmwOrderings: +% for failorder in loadOrderings: +suite.test("UnsafeAtomic<${type}>.${operation}.${successorder}.${failorder}") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + + let v = UnsafeAtomic<${type}>.create(initialValue: 0) + defer { v.destroy() } + + var (exchanged, original) = v.${operation}( + expected: 0, + desired: 1, + successOrdering: .${successorder}, + failureOrdering: .${failorder}) + expectTrue(exchanged) + expectEqual(original, 0) + expectEqual(v.load(ordering: .relaxed), 1) + + (exchanged, original) = v.${operation}( + expected: 0, + desired: 2, + successOrdering: .${successorder}, + failureOrdering: .${failorder}) + expectFalse(exchanged) + expectEqual(original, 1) + expectEqual(v.load(ordering: .relaxed), 1) + + (exchanged, original) = v.${operation}( + expected: 1, + desired: 2, + successOrdering: .${successorder}, + failureOrdering: .${failorder}) + expectTrue(exchanged) + expectEqual(original, 1) + expectEqual(v.load(ordering: .relaxed), 2) +} +% end +% end + +% for (name, _, operator, label, _) in integerOperations: +% for order in rmwOrderings: +suite.test("UnsafeAtomic<${type}>.loadThen${name}.${order}") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + + let a: ${type} = 3 + let b: ${type} = 8 + let c: ${type} = 12 + let result1: ${type} = a ${operator} b + let result2: ${type} = result1 ${operator} c + + let v = UnsafeAtomic<${type}>.create(initialValue: a) + defer { v.destroy() } + + let old1 = v.loadThen${name}(${argLabel(label)}b, ordering: .${order}) + expectEqual(old1, a) + expectEqual(v.load(ordering: .relaxed), result1) + + let old2 = v.loadThen${name}(${argLabel(label)}c, ordering: .${order}) + expectEqual(old2, result1) + expectEqual(v.load(ordering: .relaxed), result2) +} +% end +% end + +% for (name, _, operator, label, _) in integerOperations: +% for order in rmwOrderings: +suite.test("UnsafeAtomic<${type}>.${lowerFirst(name)}ThenLoad.${order}") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + + let a: ${type} = 3 + let b: ${type} = 8 + let c: ${type} = 12 + let result1: ${type} = a ${operator} b + let result2: ${type} = result1 ${operator} c + + let v = UnsafeAtomic<${type}>.create(initialValue: a) + defer { v.destroy() } + + let new1 = v.${lowerFirst(name)}ThenLoad(${argLabel(label)}b, ordering: .${order}) + expectEqual(new1, result1) + expectEqual(v.load(ordering: .relaxed), result1) + + let new2 = v.${lowerFirst(name)}ThenLoad(${argLabel(label)}c, ordering: .${order}) + expectEqual(new2, result2) + expectEqual(v.load(ordering: .relaxed), result2) +} +% end +% end + +% end diff --git a/test/Atomics/UnsafeAtomicLazyReference.swift b/test/Atomics/UnsafeAtomicLazyReference.swift new file mode 100644 index 0000000000000..964799e6c0237 --- /dev/null +++ b/test/Atomics/UnsafeAtomicLazyReference.swift @@ -0,0 +1,40 @@ +// RUN: %target-run-simple-swift +// REQUIRES: executable_test + +import StdlibUnittest +import Atomics + +let suite = TestSuite("UnsafeAtomicLazyReference") +defer { runAllTests() } + +suite.test("UnsafeAtomicLazyReference<${type}>.create-destroy") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + + let v = UnsafeAtomicLazyReference.create() + defer { v.destroy() } + expectNil(v.load()) +} + +suite.test("UnsafeAtomicLazyReference<${type}>.storeIfNilThenLoad") { + guard #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) else { + return + } + + do { + let v = UnsafeAtomicLazyReference.create() + expectNil(v.load()) + + let ref = LifetimeTracked(42) + expectTrue(v.storeIfNilThenLoad(ref) === ref) + expectTrue(v.load() === ref) + + let ref2 = LifetimeTracked(23) + expectTrue(v.storeIfNilThenLoad(ref2) === ref) + expectTrue(v.load() === ref) + + v.destroy() + } + expectEqual(LifetimeTracked.instances, 0) +}