diff --git a/stdlib/public/core/StringStorage.swift b/stdlib/public/core/StringStorage.swift index e0b2e9f723454..4e245ba8f8c0c 100644 --- a/stdlib/public/core/StringStorage.swift +++ b/stdlib/public/core/StringStorage.swift @@ -16,6 +16,7 @@ internal protocol _AbstractStringStorage: _NSCopying { var asString: String { get } + var utf16: String.UTF16View { get } var count: Int { get } var isASCII: Bool { get } var start: UnsafePointer { get } @@ -26,6 +27,7 @@ internal protocol _AbstractStringStorage: _NSCopying { internal protocol _AbstractStringStorage { var asString: String { get } + var utf16: String.UTF16View { get } var count: Int { get } var isASCII: Bool { get } var start: UnsafePointer { get } @@ -295,6 +297,10 @@ final internal class __StringStorage get { String(_StringGuts(self)) } } + @inline(__always) + final internal var utf16: String.UTF16View { + String.UTF16View(_StringGuts(self)) + } private init(_doNotCallMe: ()) { _internalInvariantFailure("Use the create method") @@ -721,6 +727,11 @@ final internal class __SharedStringStorage return String(_StringGuts(self)) } } + + @inline(__always) + final internal var utf16: String.UTF16View { + String.UTF16View(_StringGuts(self)) + } internal init( _mortal ptr: UnsafePointer, diff --git a/stdlib/public/core/StringStorageBridge.swift b/stdlib/public/core/StringStorageBridge.swift index 8d9a6903f6738..b5a6c761bb13b 100644 --- a/stdlib/public/core/StringStorageBridge.swift +++ b/stdlib/public/core/StringStorageBridge.swift @@ -43,25 +43,17 @@ extension String { // ObjC interfaces. extension _AbstractStringStorage { + @inline(__always) @_effects(releasenone) internal func _getCharacters( _ buffer: UnsafeMutablePointer, _ aRange: _SwiftNSRange ) { - _precondition(aRange.location >= 0 && aRange.length >= 0, - "Range out of bounds") - // Note: `count` is counting UTF-8 code units, while `aRange` is measured in - // UTF-16 offsets. This precondition is a necessary, but not sufficient test - // for validity. (More precise checks are done in UTF16View._nativeCopy.) - _precondition(aRange.location + aRange.length <= Int(count), - "Range out of bounds") - let range = unsafe Range( _uncheckedBounds: (aRange.location, aRange.location+aRange.length)) - let str = asString - unsafe str._copyUTF16CodeUnits( + unsafe utf16._nativeCopy( into: UnsafeMutableBufferPointer(start: buffer, count: range.count), - range: range) + offsetRange: range) } @inline(__always) @@ -116,6 +108,26 @@ extension _AbstractStringStorage { return _cocoaLengthOfBytesInEncodingTrampoline(self, encoding) } } + + // The caller info isn't useful here anyway because it's never client code, + // so this makes sure that _character(at:) doesn't have inlined assertion bits + @inline(never) + internal func _characterAtIndexOutOfBounds() -> Never { + _preconditionFailure("String index is out of bounds") + } + + @inline(__always) + @_effects(readonly) + internal func _character(at offset: Int) -> UInt16 { + if _fastPath(isASCII) { + if (_fastPath(offset < count && offset >= 0)) { + return unsafe UInt16((start + offset).pointee) + } + _characterAtIndexOutOfBounds() + } else { + return utf16[nativeNonASCIIOffset: offset] + } + } @_effects(readonly) internal func _nativeIsEqual( @@ -176,7 +188,7 @@ extension _AbstractStringStorage { start: utf16Ptr, count: otherUTF16Length ) - return unsafe asString.utf16.elementsEqual(utf16Buffer) ? 1 : 0 + return unsafe utf16.elementsEqual(utf16Buffer) ? 1 : 0 } /* @@ -197,7 +209,7 @@ extension __StringStorage { if isASCII { return count } - return asString.utf16.count + return utf16.count } } @@ -214,8 +226,7 @@ extension __StringStorage { @objc(characterAtIndex:) @_effects(readonly) final internal func character(at offset: Int) -> UInt16 { - let str = asString - return str.utf16[str._toUTF16Index(offset)] + _character(at: offset) } @objc(getCharacters:range:) @@ -313,7 +324,7 @@ extension __SharedStringStorage { if isASCII { return count } - return asString.utf16.count + return utf16.count } } @@ -330,8 +341,7 @@ extension __SharedStringStorage { @objc(characterAtIndex:) @_effects(readonly) final internal func character(at offset: Int) -> UInt16 { - let str = asString - return str.utf16[str._toUTF16Index(offset)] + _character(at: offset) } @objc(getCharacters:range:) diff --git a/stdlib/public/core/StringUTF16View.swift b/stdlib/public/core/StringUTF16View.swift index 923476a975215..f4811c3ac98ba 100644 --- a/stdlib/public/core/StringUTF16View.swift +++ b/stdlib/public/core/StringUTF16View.swift @@ -435,6 +435,22 @@ extension String.UTF16View: BidirectionalCollection { return _foreignSubscript(position: idx) } + + internal subscript(nativeNonASCIIOffset offset: Int) -> UTF16.CodeUnit { + @_effects(releasenone) get { + let threshold = _breadcrumbStride / 2 + // Do not use breadcrumbs if directly computing the result is expected + // to be cheaper + let idx = offset < threshold ? + _index(startIndex, offsetBy: offset)._knownUTF8 : + _nativeGetIndex(for: offset) + _precondition(idx._encodedOffset < _guts.count, + "String index is out of bounds") + let scalar = _guts.fastUTF8Scalar( + startingAt: _guts.scalarAlign(idx)._encodedOffset) + return scalar.utf16[idx.transcodedOffset] + } + } } extension String.UTF16View { @@ -948,6 +964,21 @@ extension String.UTF16View { fatalError() } } + + // See _nativeCopy(into:alignedRange:), except this uses un-verified UTF16 + // offsets instead of aligned indexes + internal func _nativeCopy( + into buffer: UnsafeMutableBufferPointer, + offsetRange range: Range + ) { + let alignedRange = _indexRange(for: range, from: startIndex) + _precondition(alignedRange.lowerBound._encodedOffset <= _guts.count && + alignedRange.upperBound._encodedOffset <= _guts.count, + "String index is out of bounds") + unsafe _nativeCopy( + into: buffer, + alignedRange: alignedRange.lowerBound ..< alignedRange.upperBound) + } // Copy (i.e. transcode to UTF-16) our contents into a buffer. `alignedRange` // means that the indices are part of the UTF16View.indices -- they are either @@ -962,16 +993,16 @@ extension String.UTF16View { range.lowerBound == _utf16AlignNativeIndex(range.lowerBound)) _internalInvariant( range.upperBound == _utf16AlignNativeIndex(range.upperBound)) - + if _slowPath(range.isEmpty) { return } - + let isASCII = _guts.isASCII return unsafe _guts.withFastUTF8 { utf8 in var writeIdx = 0 let writeEnd = buffer.count var readIdx = range.lowerBound._encodedOffset let readEnd = range.upperBound._encodedOffset - + if isASCII { _internalInvariant(range.lowerBound.transcodedOffset == 0) _internalInvariant(range.upperBound.transcodedOffset == 0) @@ -984,7 +1015,7 @@ extension String.UTF16View { } return } - + // Handle mid-transcoded-scalar initial index if _slowPath(range.lowerBound.transcodedOffset != 0) { _internalInvariant(range.lowerBound.transcodedOffset == 1) @@ -995,7 +1026,7 @@ extension String.UTF16View { readIdx &+= len writeIdx &+= 1 } - + // Transcode middle while readIdx < readEnd { let (scalar, len) = unsafe _decodeScalar(utf8, startingAt: readIdx) @@ -1009,13 +1040,13 @@ extension String.UTF16View { writeIdx &+= 1 } } - + // Handle mid-transcoded-scalar final index if _slowPath(range.upperBound.transcodedOffset == 1) { _internalInvariant(writeIdx < writeEnd) let (scalar, _) = unsafe _decodeScalar(utf8, startingAt: readIdx) _internalInvariant(scalar.utf16.count == 2) - + // Note: this is intentionally not using the _unchecked subscript. // (We rely on debug assertions to catch out of bounds access.) unsafe buffer[writeIdx] = scalar.utf16[0] diff --git a/stdlib/public/core/UnicodeHelpers.swift b/stdlib/public/core/UnicodeHelpers.swift index 79f8af7c7f83a..47fde997f9abb 100644 --- a/stdlib/public/core/UnicodeHelpers.swift +++ b/stdlib/public/core/UnicodeHelpers.swift @@ -61,7 +61,7 @@ internal func _decodeUTF8( return Unicode.Scalar(_unchecked: value) } -@inlinable +@inlinable @inline(__always) internal func _decodeScalar( _ utf8: UnsafeBufferPointer, startingAt i: Int ) -> (Unicode.Scalar, scalarLength: Int) { @@ -207,7 +207,7 @@ extension _StringGuts { } } - @inlinable + @inlinable @inline(__always) internal func fastUTF8Scalar(startingAt i: Int) -> Unicode.Scalar { _internalInvariant(isFastUTF8) return unsafe self.withFastUTF8 { unsafe _decodeScalar($0, startingAt: i).0 } diff --git a/stdlib/public/core/UnsafeBufferPointer.swift.gyb b/stdlib/public/core/UnsafeBufferPointer.swift.gyb index ddf6b70ee85b4..b37c7d0b5e4aa 100644 --- a/stdlib/public/core/UnsafeBufferPointer.swift.gyb +++ b/stdlib/public/core/UnsafeBufferPointer.swift.gyb @@ -212,7 +212,7 @@ extension Unsafe${Mutable}BufferPointer: @unsafe Sequence { return (unsafe Iterator(_position: s + n, _end: s + count), n) } - @inlinable + @inlinable @_transparent @safe public func withContiguousStorageIfAvailable( _ body: (UnsafeBufferPointer) throws -> R @@ -780,7 +780,7 @@ extension Unsafe${Mutable}BufferPointer: return try unsafe body(&self) } - @inlinable + @inlinable @_transparent @safe public mutating func withContiguousMutableStorageIfAvailable( _ body: (inout UnsafeMutableBufferPointer) throws -> R @@ -825,7 +825,7 @@ extension Unsafe${Mutable}BufferPointer { /// - `rebased.count == slice.count` /// /// - Parameter slice: The buffer slice to rebase. - @inlinable // unsafe-performance + @inlinable @_transparent // unsafe-performance public init(rebasing slice: Slice>) { // NOTE: `Slice` does not guarantee that its start/end indices are valid // in `base` -- it merely ensures that `startIndex <= endIndex`. @@ -864,7 +864,7 @@ extension Unsafe${Mutable}BufferPointer { /// - `rebased.count == slice.count` /// /// - Parameter slice: The buffer slice to rebase. - @inlinable // unsafe-performance + @inlinable @_transparent // unsafe-performance public init(rebasing slice: Slice>) { let base = unsafe slice.base.baseAddress?.advanced(by: slice.startIndex) let count = unsafe slice.endIndex &- slice.startIndex @@ -993,7 +993,7 @@ extension UnsafeMutableBufferPointer { /// initialize the buffer's storage. /// - Returns: The index one past the last element of the buffer initialized /// by this function. - @_alwaysEmitIntoClient + @_alwaysEmitIntoClient @_transparent public func initialize( fromContentsOf source: some Collection ) -> Index { @@ -1418,7 +1418,7 @@ extension Unsafe${Mutable}BufferPointer where Element: ~Copyable { /// method. /// - buffer: The buffer temporarily bound to `T`. /// - Returns: The return value, if any, of the `body` closure parameter. - @_alwaysEmitIntoClient + @_alwaysEmitIntoClient @_transparent public func withMemoryRebound( to type: T.Type, _ body: (_ buffer: ${Self}) throws(E) -> Result diff --git a/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb b/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb index d6e4b7da6bb04..e76d6a1a6fd16 100644 --- a/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb +++ b/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb @@ -102,7 +102,7 @@ public struct Unsafe${Mutable}RawBufferPointer { // This works around _debugPrecondition() impacting the performance of // optimized code. (rdar://72246338) - @_alwaysEmitIntoClient + @_alwaysEmitIntoClient @_transparent internal init( @_nonEphemeral _uncheckedStart start: Unsafe${Mutable}RawPointer?, count: Int @@ -120,7 +120,7 @@ public struct Unsafe${Mutable}RawBufferPointer { /// for a non-`nil` `start`. /// - count: The number of bytes to include in the buffer. `count` must not /// be negative. - @inlinable + @inlinable @_transparent public init( @_nonEphemeral start: Unsafe${Mutable}RawPointer?, count: Int ) { @@ -130,7 +130,7 @@ public struct Unsafe${Mutable}RawBufferPointer { unsafe self.init(_uncheckedStart: start, count: count) } - @safe + @safe @_transparent @_alwaysEmitIntoClient public init(_empty: ()) { unsafe _position = nil @@ -150,7 +150,7 @@ extension UnsafeRawBufferPointer { @usableFromInline internal var _position, _end: UnsafeRawPointer? - @inlinable + @inlinable @_transparent internal init(_position: UnsafeRawPointer?, _end: UnsafeRawPointer?) { unsafe self._position = _position unsafe self._end = _end @@ -621,7 +621,7 @@ extension Unsafe${Mutable}RawBufferPointer { /// Creates a new buffer over the same memory as the given buffer. /// /// - Parameter bytes: The buffer to convert. - @inlinable + @inlinable @_transparent @safe public init(_ bytes: UnsafeMutableRawBufferPointer) { unsafe self.init(start: bytes.baseAddress, count: bytes.count) @@ -631,7 +631,7 @@ extension Unsafe${Mutable}RawBufferPointer { /// Creates a new mutable buffer over the same memory as the given buffer. /// /// - Parameter bytes: The buffer to convert. - @inlinable + @inlinable @_transparent public init(mutating bytes: UnsafeRawBufferPointer) { unsafe self.init(start: UnsafeMutableRawPointer(mutating: bytes.baseAddress), count: bytes.count) @@ -640,7 +640,7 @@ extension Unsafe${Mutable}RawBufferPointer { /// Creates a new buffer over the same memory as the given buffer. /// /// - Parameter bytes: The buffer to convert. - @inlinable + @inlinable @_transparent @safe public init(_ bytes: UnsafeRawBufferPointer) { unsafe self.init(start: bytes.baseAddress, count: bytes.count) @@ -651,7 +651,7 @@ extension Unsafe${Mutable}RawBufferPointer { /// /// - Parameter buffer: The typed buffer to convert to a raw buffer. The /// buffer's type `T` must be a trivial type. - @inlinable + @inlinable @_transparent @_preInverseGenerics @safe public init(_ buffer: UnsafeMutableBufferPointer) { @@ -664,7 +664,7 @@ extension Unsafe${Mutable}RawBufferPointer { /// /// - Parameter buffer: The typed buffer to convert to a raw buffer. The /// buffer's type `T` must be a trivial type. - @inlinable + @inlinable @_transparent @_preInverseGenerics @safe public init(_ buffer: UnsafeBufferPointer) { @@ -694,7 +694,7 @@ extension Unsafe${Mutable}RawBufferPointer { /// - `rebased.count == slice.count` /// /// - Parameter slice: The raw buffer slice to rebase. - @inlinable + @inlinable @_transparent public init(rebasing slice: Slice) { // NOTE: `Slice` does not guarantee that its start/end indices are valid // in `base` -- it merely ensures that `startIndex <= endIndex`. @@ -733,7 +733,7 @@ extension Unsafe${Mutable}RawBufferPointer { /// - `rebased.count == slice.count` /// /// - Parameter slice: The raw buffer slice to rebase. - @inlinable + @inlinable @_transparent public init(rebasing slice: Slice) { let base = unsafe slice.base.baseAddress?.advanced(by: slice.startIndex) let count = unsafe slice.endIndex &- slice.startIndex @@ -1110,7 +1110,7 @@ extension Unsafe${Mutable}RawBufferPointer { /// the return value for the `withMemoryRebound(to:capacity:_:)` method. /// - buffer: The buffer temporarily bound to instances of `T`. /// - Returns: The return value, if any, of the `body` closure parameter. - @_alwaysEmitIntoClient + @_alwaysEmitIntoClient @_transparent public func withMemoryRebound( to type: T.Type, _ body: (_ buffer: Unsafe${Mutable}BufferPointer) throws(E) -> Result @@ -1147,7 +1147,7 @@ extension Unsafe${Mutable}RawBufferPointer { /// /// - Parameter to: The type `T` that the memory has already been bound to. /// - Returns: A typed pointer to the same memory as this raw pointer. - @_alwaysEmitIntoClient + @_alwaysEmitIntoClient @_transparent public func assumingMemoryBound( to: T.Type ) -> Unsafe${Mutable}BufferPointer { @@ -1163,7 +1163,7 @@ extension Unsafe${Mutable}RawBufferPointer { % if Mutable: @_alwaysEmitIntoClient - @inline(__always) + @_transparent public func withContiguousMutableStorageIfAvailable( _ body: (inout UnsafeMutableBufferPointer) throws -> R ) rethrows -> R? { @@ -1181,7 +1181,7 @@ extension Unsafe${Mutable}RawBufferPointer { % end @_alwaysEmitIntoClient - @inline(__always) + @_transparent public func withContiguousStorageIfAvailable( _ body: (UnsafeBufferPointer) throws -> R ) rethrows -> R? { diff --git a/validation-test/stdlib/UnsafeBufferPointer.swift.gyb b/validation-test/stdlib/UnsafeBufferPointer.swift.gyb index ada9a8f6277d2..a5566bccb2559 100644 --- a/validation-test/stdlib/UnsafeBufferPointer.swift.gyb +++ b/validation-test/stdlib/UnsafeBufferPointer.swift.gyb @@ -287,8 +287,11 @@ ${SelfName}TestSuite.test("badCount") let emptyAllocated = UnsafeMutablePointer.allocate(capacity: 0) defer { emptyAllocated.deallocate() } + + // Make sure we can't emit the error at compile time + @inline(never) var badCount: Int { -1 } - let buffer = ${SelfType}(start: ${PointerType}(emptyAllocated), count: -1) + let buffer = ${SelfType}(start: ${PointerType}(emptyAllocated), count: badCount) _ = buffer }