Make double-wide atomics conditional on platform support

Linux on x86_64 unfortunately disables support for cmpxchg16 by default, making double-wide atomic operations fall back to using locks. We cannot currently override this on the package level, so stop providing double-wide atomics in this configuration by default. To enable codegen support for double-wide atomics on Linux/x86_64, compile the package with `-Xcc -mcx16 -Xswiftc -DENABLE_DOUBLEWIDE_ATOMICS`.
lorentey · Oct 9, 2020 · 9ad914d · 9ad914d
1 parent 56753d1
commit 9ad914d
Show file tree

Hide file tree

Showing 11 changed files with 7,654 additions and 7,621 deletions.
diff --git a/Package.swift b/Package.swift
@@ -24,8 +24,7 @@ let package = Package(
     .target(name: "_AtomicsShims"),
     .target(
       name: "Atomics",
-      dependencies: ["_AtomicsShims"],
-      path: "Sources/Atomics"
+      dependencies: ["_AtomicsShims"]
     ),
     .testTarget(
       name: "AtomicsTests",

diff --git a/Sources/Atomics/AtomicStrongReference.swift b/Sources/Atomics/AtomicStrongReference.swift
@@ -12,6 +12,10 @@
 
 import _AtomicsShims
 
+// Double-wide atomic primitives on x86_64 CPUs aren't available by default
+// on Linux distributions, and we cannot currently enable them automatically.
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
+
 /// A class type that supports atomic strong references.
 public protocol AtomicReference: AnyObject, AtomicOptionalWrappable
 where
@@ -569,3 +573,4 @@ extension AtomicOptionalReferenceStorage: AtomicStorage {
   }
 }
 
+#endif // ENABLE_DOUBLEWIDE_ATOMICS
diff --git a/Sources/Atomics/IntegerConformances.swift.gyb b/Sources/Atomics/IntegerConformances.swift.gyb
@@ -24,6 +24,11 @@ ${autogenerated_warning()}
 import _AtomicsShims
 
 % for swiftType in atomicTypes():
+%  if swiftType == "DoubleWord":
+// Double-wide atomic primitives on x86_64 CPUs aren't available by default
+// on Linux distributions, and we cannot currently enable them automatically.
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
+%  end
 extension ${swiftType}: AtomicValue {
   public struct AtomicRepresentation {
     public typealias Value = ${swiftType}
@@ -219,4 +224,7 @@ extension ${swiftType}.AtomicRepresentation: AtomicIntegerStorage {
 }
 % end
 
+%  if swiftType == "DoubleWord":
+#endif // ENABLE_DOUBLEWIDE_ATOMICS
+%  end
 % end
diff --git a/Sources/Atomics/autogenerated/IntegerConformances.swift b/Sources/Atomics/autogenerated/IntegerConformances.swift
@@ -4939,6 +4939,9 @@ extension UInt8.AtomicRepresentation: AtomicIntegerStorage {
   }
 }
 
+// Double-wide atomic primitives on x86_64 CPUs aren't available by default
+// on Linux distributions, and we cannot currently enable them automatically.
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
 extension DoubleWord: AtomicValue {
   public struct AtomicRepresentation {
     public typealias Value = DoubleWord
@@ -5262,3 +5265,4 @@ extension DoubleWord.AtomicRepresentation: AtomicStorage {
 }
 
 
+#endif // ENABLE_DOUBLEWIDE_ATOMICS
diff --git a/Sources/_AtomicsShims/include/_AtomicsShims.h b/Sources/_AtomicsShims/include/_AtomicsShims.h
@@ -59,6 +59,20 @@
 #include <stdatomic.h>
 #include <assert.h>
 
+// For now, assume double-wide atomics are available everywhere,
+// except on Linux/x86_64, where they need to be manually enabled
+// by the `cx16` target attribute. (Unfortunately we cannot currently
+// turn that on in our package description.)
+#ifdef __APPLE__
+#  define ENABLE_DOUBLEWIDE_ATOMICS 1
+#elif defined(_WIN32)
+#  define ENABLE_DOUBLEWIDE_ATOMICS 1
+#elif defined(__linux__)
+#  if !defined(__x86_64__) || defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)
+#    define ENABLE_DOUBLEWIDE_ATOMICS 1
+#  endif
+#endif
+
 #define SWIFTATOMIC_INLINE static inline __attribute__((__always_inline__))
 #define SWIFTATOMIC_SWIFT_NAME(name) __attribute__((swift_name(#name)))
 
@@ -320,9 +334,13 @@ _sa_dword _sa_decode_dword(__uint128_t value) {
   return result;
 }
 
+#if ENABLE_DOUBLEWIDE_ATOMICS
 #define SWIFTATOMIC_ENCODE_DoubleWord(_value) (_value).value
 #define SWIFTATOMIC_DECODE_DoubleWord(value) _sa_decode_dword(value)
 SWIFTATOMIC_DEFINE_TYPE(COMPLEX, DoubleWord, _sa_dword, __uint128_t)
+#else
+SWIFTATOMIC_STORAGE_TYPE(DoubleWord, _sa_dword, __uint128_t)
+#endif
 
 #elif __INTPTR_WIDTH__ == 32
 
@@ -352,15 +370,21 @@ _sa_dword _sa_decode_dword(uint64_t value) {
   return result;
 }
 
+#if ENABLE_DOUBLEWIDE_ATOMICS
 #define SWIFTATOMIC_ENCODE_DoubleWord(_value) (_value).value
 #define SWIFTATOMIC_DECODE_DoubleWord(value) _sa_decode_dword(value)
 SWIFTATOMIC_DEFINE_TYPE(COMPLEX, DoubleWord, _sa_dword, uint64_t)
+#else
+SWIFTATOMIC_STORAGE_TYPE(DoubleWord, _sa_dword, uint64_t)
+#endif // ENABLE_DOUBLEWIDE_ATOMICS
 
 #else
 #error "Unsupported intptr_t bit width"
 #endif // __INTPTR_WIDTH
 
+#if ENABLE_DOUBLEWIDE_ATOMICS
 extern void _sa_retain_n(void *object, uint32_t n);
 extern void _sa_release_n(void *object, uint32_t n);
+#endif
 
 #endif //SWIFTATOMIC_HEADER_INCLUDED
diff --git a/Sources/_AtomicsShims/src/_AtomicsShims.c b/Sources/_AtomicsShims/src/_AtomicsShims.c
@@ -10,14 +10,23 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <dlfcn.h>
 #include "_AtomicsShims.h"
 
-void _sa_retain_n(void *object, uint32_t n) {
+#if ENABLE_DOUBLEWIDE_ATOMICS
+// FIXME: These should be static inline header-only shims, but Swift 5.3 doesn't
+// like calls to swift_retain_n/swift_release_n appearing in Swift code, not
+// even when imported through C. (See https://bugs.swift.org/browse/SR-13708)
+
+void _sa_retain_n(void *object, uint32_t n)
+{
   extern void *swift_retain_n(void *object, uint32_t n);
   swift_retain_n(object, n);
 }
 
-void _sa_release_n(void *object, uint32_t n) {
+void _sa_release_n(void *object, uint32_t n)
+{
   extern void swift_release_n(void *object, uint32_t n);
   swift_release_n(object, n);
 }
+#endif
diff --git a/Tests/AtomicsTests/Basics.swift.gyb b/Tests/AtomicsTests/Basics.swift.gyb
@@ -28,7 +28,6 @@
     ("UInt64",     "UInt64", "12", "23"),
 
     ("Bool",       "Bool",   "true", "false"),
-    ("DoubleWord", "DoubleWord", "DoubleWord(high: 100, low: 64)", "DoubleWord(high: 50, low: 32)"),
 
     ("Pointer",                   "UnsafePointer<Foo>",         "_foo1",  "_foo2"),
     ("OptionalPointer",           "UnsafePointer<Foo>?",        "nil",    "_foo2"),
@@ -43,10 +42,12 @@
     ("Unmanaged",                 "Unmanaged<Bar>",             "_bar1", "_bar2"),
     ("OptionalUnmanaged",         "Unmanaged<Bar>?",            "nil",   "_bar2"),
 
+    ("RawRepresentable",          "Fred", "Fred.one", "Fred.two"),
+
+    ("DoubleWord", "DoubleWord", "DoubleWord(high: 100, low: 64)", "DoubleWord(high: 50, low: 32)"),
+
     ("Reference",                 "Baz",  "_baz1", "_baz2"),
     ("OptionalReference",         "Baz?", "nil",   "_baz2"),
-
-    ("RawRepresentable",          "Fred", "Fred.one", "Fred.two"),
   ]
 }%
 ${autogenerated_warning()}
@@ -75,6 +76,7 @@ private class Bar: Equatable, CustomStringConvertible {
   }
 }
 
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
 private class Baz: Equatable, CustomStringConvertible, AtomicReference {
   var value: Int
   init(_ value: Int) { self.value = value }
@@ -83,13 +85,19 @@ private class Baz: Equatable, CustomStringConvertible, AtomicReference {
     left === right
   }
 }
+#endif
 
 private enum Fred: Int, AtomicValue {
   case one
   case two
 }
 
 % for label, type, a, b in types:
+% if label == "DoubleWord" or label == "Reference" or label == "OptionalReference":
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
+% else:
+#if true
+% end
 /// Exercises all operations in a single-threaded context, verifying
 /// they provide the expected results.
 class BasicAtomic${label}Tests: XCTestCase {
@@ -278,9 +286,8 @@ class BasicAtomic${label}Tests: XCTestCase {
 % end
 
 
-// Bool operations
-
 % if type == "Bool":
+  // Bool operations
 %   for (name, _, operator, arglabel, _) in boolOperations:
 %     for (order, _, _) in updateOrderings:
   func test_loadThen${name}_${order}() {
@@ -328,9 +335,8 @@ class BasicAtomic${label}Tests: XCTestCase {
 %   end
 % end
 
-// Integer operations
-
 % if type.startswith("Int") or type.startswith("UInt"):
+// Integer operations
 %   for (name, _, operator, arglabel, _) in integerOperations:
 %     for (order, _, _) in updateOrderings:
   func test_loadThen${name}_${order}() {
@@ -379,4 +385,5 @@ class BasicAtomic${label}Tests: XCTestCase {
 % end
 
 }
+#endif
 % end
diff --git a/Tests/AtomicsTests/LockFreeQueue.swift b/Tests/AtomicsTests/LockFreeQueue.swift
@@ -28,6 +28,9 @@ import XCTest
 import Dispatch
 import Atomics
 
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
+private let nodeCount = ManagedAtomic<Int>(0)
+
 class LockFreeQueue<Element> {
   class Node: AtomicReference {
     let next: ManagedAtomic<Node?>
@@ -36,6 +39,7 @@ class LockFreeQueue<Element> {
     init(value: Element?, next: Node?) {
       self.value = value
       self.next = ManagedAtomic(next)
+      nodeCount.wrappingIncrement(ordering: .relaxed)
     }
 
     deinit {
@@ -52,6 +56,7 @@ class LockFreeQueue<Element> {
       if values > 0 {
         print(values)
       }
+      nodeCount.wrappingDecrement(ordering: .relaxed)
     }
   }
 
@@ -128,6 +133,10 @@ class LockFreeQueue<Element> {
 }
 
 class QueueTests: XCTestCase {
+  override func tearDown() {
+    XCTAssertEqual(nodeCount.load(ordering: .relaxed), 0)
+  }
+
   func check(readers: Int, writers: Int, count: Int) {
     let queue = LockFreeQueue<(writer: Int, value: Int)>()
     let num = ManagedAtomic(0)
@@ -169,3 +178,4 @@ class QueueTests: XCTestCase {
   }
 }
 
+#endif
diff --git a/Tests/AtomicsTests/StrongReferenceRace.swift b/Tests/AtomicsTests/StrongReferenceRace.swift
@@ -14,6 +14,7 @@ import XCTest
 import Atomics
 import Dispatch
 
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
 let iterations = 1_000_000
 
 private class Node: AtomicReference {
@@ -227,3 +228,4 @@ class StrongReferenceRace: XCTestCase {
   func testLifetimes_08() { checkLifetimes(count: 8, iterations: iterations) }
   func testLifetimes_16() { checkLifetimes(count: 16, iterations: iterations) }
 }
+#endif
diff --git a/Tests/AtomicsTests/StrongReferenceShuffle.swift b/Tests/AtomicsTests/StrongReferenceShuffle.swift
@@ -19,7 +19,8 @@ import XCTest
 import Atomics
 import Dispatch
 
-let nodeCount = ManagedAtomic<Int>(0)
+#if !(os(Linux) && arch(x86_64)) || ENABLE_DOUBLEWIDE_ATOMICS
+private let nodeCount = ManagedAtomic<Int>(0)
 
 private class List<Value: Equatable> {
   class Node: AtomicReference {
@@ -256,3 +257,4 @@ class StrongReferenceShuffleTests: XCTestCase {
   func test_sink_04_08() { checkSink(writers: 4, readers: 8, iterations: iterations) }
   func test_sink_08_08() { checkSink(writers: 8, readers: 8, iterations: iterations) }
 }
+#endif