diff --git a/stdlib/public/core/KeyPath.swift b/stdlib/public/core/KeyPath.swift index cda7e2cc4db5c..251e95467086c 100644 --- a/stdlib/public/core/KeyPath.swift +++ b/stdlib/public/core/KeyPath.swift @@ -24,6 +24,11 @@ internal func _abstract( #endif } +@_alignment(16) +struct _SixteenAligned { + let x: UInt8 +} + // MARK: Type-erased abstract base classes // NOTE: older runtimes had Swift.AnyKeyPath as the ObjC name. @@ -154,8 +159,37 @@ public class AnyKeyPath: _AppendKeyPath { ) -> Self { _internalInvariant(bytes > 0 && bytes % 4 == 0, "capacity must be multiple of 4 bytes") - let result = Builtin.allocWithTailElems_1(self, (bytes/4)._builtinWordValue, - Int32.self) + + // Subtract the buffer header and any padding it may have from the number of + // bytes we need to allocate. The alloc below has a 0 sized 16 byte aligned + // tail element that will force the compiler to insert buffer header + any + // padding necessary to accomodate. + let bytesWithoutHeader = bytes &- MemoryLayout.size + + let result = Builtin.allocWithTailElems_2( + self, + + // This tail element accomplishes two things: + // 1. Guarantees a 16 byte alignment for the allocated object pointer. + // 2. Forces the compiler to add padding after the kvc string to support + // this 16 byte aligned tail element. + // + // The size of keypath objects before any tail elements is 24 bytes large + // on 64 bit platforms and 12 bytes large on 32 bit platforms. + // (Isa, RC, and KVC). The amount of padding bytes needed after the kvc + // string pointer to support a 16 byte aligned tail element is the exact + // same amount of bytes that the keypath buffer header occupies on both + // 64 & 32 bit platforms (because we always start component headers on + // pointer aligned addresses). That is the purpose for the subtraction + // above. The result of this tail element should just be the 16 byte + // pointer aligned requirement and no extra bytes allocated. + 0._builtinWordValue, + _SixteenAligned.self, + + (bytesWithoutHeader/4)._builtinWordValue, + Int32.self + ) + unsafe result._kvcKeyPathStringPtr = nil let base = UnsafeMutableRawPointer(Builtin.projectTailElems(result, Int32.self)) @@ -1119,6 +1153,163 @@ internal enum KeyPathComputedIDResolution { case functionCall } +@_unavailableInEmbedded +internal struct ComputedArgumentSize { + var value: UInt + + @_unavailableInEmbedded + static var sizeMask: UInt { +#if _pointerBitWidth(_64) + 0x3FFF_FFFF_FFFF_FFFF +#elseif _pointerBitWidth(_32) + 0xFFF_FFFF +#else +#warning("Unsupported platform") + fatalError() +#endif + } + + @_unavailableInEmbedded + static var paddingMask: UInt { +#if _pointerBitWidth(_64) + 0x4000_0000_0000_0000 +#elseif _pointerBitWidth(_32) + 0x3000_0000 +#else +#warning("Unsupported platform") + fatalError() +#endif + } + + @_unavailableInEmbedded + static var paddingShift: UInt { +#if _pointerBitWidth(_64) + 62 +#elseif _pointerBitWidth(_32) + 28 +#else +#warning("Unsupported platform") + fatalError() +#endif + } + + @_unavailableInEmbedded + static var alignmentMask: UInt { +#if _pointerBitWidth(_64) + 0x8000_0000_0000_0000 +#elseif _pointerBitWidth(_32) + 0x6000_0000 +#else +#warning("Unsupported platform") + fatalError() +#endif + } + + @_unavailableInEmbedded + static var alignmentShift: UInt { +#if _pointerBitWidth(_64) + 63 +#elseif _pointerBitWidth(_32) + 30 +#else +#warning("Unsupported platform") + fatalError() +#endif + } + + init(_ argSize: Int) { + value = UInt(truncatingIfNeeded: argSize) + } + + // The size of just the arguments only, ignoring padding. + var argumentSize: Int { + Int(truncatingIfNeeded: value & Self.sizeMask) + } + + // The total size of the argument buffer. + var totalSize: Int { + argumentSize &+ padding + } + + // The number of padding bytes required so that the argument is properly + // aligned in the keypath buffer. This is in the top 3rd and 4th bits for 32 + // bit platforms (because they need to represent 8 and 16 overaligned types) + // and top 2nd bit for 64 bit platforms (because they only need to represent 16 + // alignment). + var padding: Int { + get { + let mask = value & Self.paddingMask + let shift = mask &>> Self.paddingShift + return Int(truncatingIfNeeded: shift) &* MemoryLayout.size + } + + set { + let new = UInt(truncatingIfNeeded: newValue) + let reduced = new / UInt(truncatingIfNeeded: MemoryLayout.size) + let shift = reduced &<< Self.paddingShift + value &= ~Self.paddingMask + value |= shift + } + } + + // If the arguments have an alignment greater than the platform's pointer + // alignment, then this structure records the argument's alignment. Even if + // the argument didn't require padding, we still record the argument's + // alignment here to prevent overallocation on keypath appends (because a leaf + // component may suddenly need padding or lose padding). This returns 0 if the + // argument had less than or equal to the platform's pointer alignment. The + // only valid values of this on 64 bit are 16 and 0, while on 32 it is 16, 8, + // and 0. Top bit on 64 bit platforms and top 2 bits on 32 bit platforms. + @_unavailableInEmbedded + var alignment: Int { + get { + let mask = value & Self.alignmentMask + let shift = mask &>> Self.alignmentShift + +#if _pointerBitWidth(_64) + // On 64 bit, the only higher alignment a type could have is 16 byte. + return shift == 1 ? 16 : 0 +#elseif _pointerBitWidth(_32) + switch shift { + case 1: + return 8 + case 2: + return 16 + default: + return 0 + } +#else +#warning("Unsupported platform") + fatalError() +#endif + } + + set { + var reduced: UInt = 0 + +#if _pointerBitWidth(_64) + reduced = newValue == 16 ? 1 : 0 +#elseif _pointerBitWidth(_32) + switch newValue { + case 8: + reduced = 1 + case 16: + reduced = 2 + default: + reduced = 0 + } +#else +#warning("Unsupported platform") + fatalError() +#endif + + let shift = reduced &<< Self.alignmentShift + value &= ~Self.alignmentMask + value |= shift + } + } +} + @_unavailableInEmbedded @safe internal struct RawKeyPathComponent { @@ -1534,7 +1725,7 @@ internal struct RawKeyPathComponent { // two words for argument header: size, witnesses total &+= ptrSize &* 2 // size of argument area - total &+= _computedArgumentSize + total &+= _computedArgumentSize.totalSize if header.isComputedInstantiatedFromExternalWithArguments { total &+= Header.externalWithArgumentsExtraSize } @@ -1592,8 +1783,8 @@ internal struct RawKeyPathComponent { (header.isComputedSettable ? 3 : 2) } - internal var _computedArgumentSize: Int { - return unsafe _computedArgumentHeaderPointer.load(as: Int.self) + internal var _computedArgumentSize: ComputedArgumentSize { + return unsafe _computedArgumentHeaderPointer.load(as: ComputedArgumentSize.self) } internal var _computedArgumentWitnesses: ComputedArgumentWitnessesPtr { @@ -1611,6 +1802,10 @@ internal struct RawKeyPathComponent { if header.isComputedInstantiatedFromExternalWithArguments { unsafe base += Header.externalWithArgumentsExtraSize } + + // If the argument needed padding to be properly aligned, skip that. + unsafe base += _computedArgumentSize.padding + return unsafe base } internal var _computedMutableArguments: UnsafeMutableRawPointer { @@ -1648,7 +1843,7 @@ internal struct RawKeyPathComponent { if header.hasComputedArguments { unsafe argument = unsafe KeyPathComponent.ArgumentRef( data: UnsafeRawBufferPointer(start: _computedArguments, - count: _computedArgumentSize), + count: _computedArgumentSize.argumentSize), witnesses: _computedArgumentWitnesses, witnessSizeAdjustment: _computedArgumentWitnessSizeAdjustment) } else { @@ -1688,15 +1883,18 @@ internal struct RawKeyPathComponent { if header.hasComputedArguments, let destructor = unsafe _computedArgumentWitnesses.destroy { unsafe destructor(_computedMutableArguments, - _computedArgumentSize &- _computedArgumentWitnessSizeAdjustment) + _computedArgumentSize.argumentSize &- _computedArgumentWitnessSizeAdjustment) } case .external: _internalInvariantFailure("should have been instantiated away") } } - internal func clone(into buffer: inout UnsafeMutableRawBufferPointer, - endOfReferencePrefix: Bool) { + internal func clone( + into buffer: inout UnsafeMutableRawBufferPointer, + endOfReferencePrefix: Bool, + adjustForAlignment: Bool + ) { var newHeader = header newHeader.endOfReferencePrefix = endOfReferencePrefix @@ -1744,10 +1942,23 @@ internal struct RawKeyPathComponent { if header.hasComputedArguments { let arguments = unsafe _computedArguments let argumentSize = _computedArgumentSize - unsafe buffer.storeBytes(of: argumentSize, - toByteOffset: componentSize, - as: Int.self) + // Remember the argument size location, we may need to adjust the value + // for alignment. + let argumentSizePtr = unsafe buffer.baseAddress._unsafelyUnwrappedUnchecked + componentSize + + // If we don't need to adjust for alignment, then the current argument + // size is correct. If alignment is 0, then the arguments had equal + // to or less than alignment to the platform's word alignment. + if !adjustForAlignment || argumentSize.alignment == 0 { + unsafe buffer.storeBytes( + of: argumentSize, + toByteOffset: componentSize, + as: ComputedArgumentSize.self + ) + } + componentSize += MemoryLayout.size + unsafe buffer.storeBytes(of: _computedArgumentWitnesses, toByteOffset: componentSize, as: ComputedArgumentWitnessesPtr.self) @@ -1761,13 +1972,56 @@ internal struct RawKeyPathComponent { as: Int.self) componentSize += MemoryLayout.size } - let adjustedSize = argumentSize - _computedArgumentWitnessSizeAdjustment - let argumentDest = + + let adjustedSize = argumentSize.argumentSize &- + _computedArgumentWitnessSizeAdjustment + var argumentDest = unsafe buffer.baseAddress.unsafelyUnwrapped + componentSize + + // If we're not adjusting for alignment, but we had existing padding, we + // need to mimic the exact padding. + if !adjustForAlignment, argumentSize.padding != 0 { + unsafe argumentDest += argumentSize.padding + componentSize &+= argumentSize.padding + } + + // We've been asked to adjust for alignment and the original argument + // was overaligned. Check for misalignment at our current destination. + if adjustForAlignment, argumentSize.alignment != 0 { + var newArgumentSize = ComputedArgumentSize(argumentSize.argumentSize) + let argumentAlignmentMask = argumentSize.alignment &- 1 + let misaligned = Int(bitPattern: argumentDest) & argumentAlignmentMask + + if misaligned == 0 { + newArgumentSize.padding = 0 + newArgumentSize.alignment = argumentSize.alignment + + unsafe argumentSizePtr.storeBytes( + of: newArgumentSize, + as: ComputedArgumentSize.self + ) + } else { + // Otherwise, this is still misaligned and we need to calculate + // how much padding we need. + newArgumentSize.padding = argumentSize.alignment &- misaligned + newArgumentSize.alignment = argumentSize.alignment + + unsafe argumentSizePtr.storeBytes( + of: newArgumentSize, + as: ComputedArgumentSize.self + ) + + // Push the buffer past the padding. + unsafe argumentDest += newArgumentSize.padding + componentSize += newArgumentSize.padding + } + } + unsafe _computedArgumentWitnesses.copy( arguments, argumentDest, adjustedSize) + if header.isComputedInstantiatedFromExternalWithArguments { // The extra information for external property descriptor arguments // can always be memcpy'd. @@ -1776,7 +2030,7 @@ internal struct RawKeyPathComponent { size: UInt(_computedArgumentWitnessSizeAdjustment)) } - componentSize += argumentSize + componentSize += argumentSize.argumentSize } case .external: @@ -2160,6 +2414,7 @@ internal struct KeyPathBuffer { } } + @inline(never) internal mutating func next() -> (RawKeyPathComponent, Any.Type?) { let header = unsafe _pop(from: &data, as: RawKeyPathComponent.Header.self) // Track if this is the last component of the reference prefix. @@ -2610,6 +2865,101 @@ internal func _tryToAppendKeyPaths( return _openExistential(rootRoot, do: open) } +@_unavailableInEmbedded +internal func calculateAppendedKeyPathSize( + _ root: AnyKeyPath, + _ leaf: AnyKeyPath, + _ rootBuffer: KeyPathBuffer, + _ leafBuffer: KeyPathBuffer +) -> (Int, Int, Int, Int, Int, Int) { + var result = MemoryLayout.size // Header size (padding if needed) + var resultWithObjectHeaderAndKVC: Int { + result &+ MemoryLayout.size &* 3 + } + + // Result buffer has room for both key paths' components, plus the + // header, plus space for the middle type. + // Align up the root so that we can put the component type after it. + result &+= unsafe MemoryLayout._roundingUpToAlignment(rootBuffer.data.count) + result &+= MemoryLayout.size // Middle type + + var leafIter = unsafe leafBuffer + while true { + let (component, nextType) = unsafe leafIter.next() + let isLast = nextType == nil + + result &+= MemoryLayout.size + + if component.header.kind == .computed, + component.header.hasComputedArguments, + component._computedArgumentSize.alignment != 0 { + result = MemoryLayout._roundingUpToAlignment(result) + + // Id and getter + result &+= MemoryLayout.size &* 2 + + if component.header.isComputedSettable { + result &+= MemoryLayout.size + } + + // Argument size and witness table + result &+= MemoryLayout.size &* 2 + + // If we also have external arguments, we need to store the size of the + // local arguments so we can find the external component arguments. + if component.header.isComputedInstantiatedFromExternalWithArguments { + result &+= RawKeyPathComponent.Header.externalWithArgumentsExtraSize + } + + let alignmentMask = component._computedArgumentSize.alignment &- 1 + let misaligned = resultWithObjectHeaderAndKVC & alignmentMask + + if misaligned != 0 { + result &+= component._computedArgumentSize.alignment &- misaligned + } + + result &+= component._computedArgumentSize.argumentSize + } else { + result &+= component.bodySize + } + + if isLast { + break + } + } + + // Size of just our components is equal to root + middle + leaf (get rid of + // the header and any padding needed). + let componentSize = result &- MemoryLayout.size + + // The first member after the components is the maxSize of the keypath. + result = MemoryLayout._roundingUpToAlignment(result) + result &+= MemoryLayout.size + + // Reserve room for the appended KVC string, if both key paths are + // KVC-compatible. + let appendedKVCLength: Int, rootKVCLength: Int, leafKVCLength: Int + + if root.getOffsetFromStorage() == nil, leaf.getOffsetFromStorage() == nil, + let rootPtr = unsafe root._kvcKeyPathStringPtr, + let leafPtr = unsafe leaf._kvcKeyPathStringPtr { + rootKVCLength = unsafe Int(_swift_stdlib_strlen(rootPtr)) + leafKVCLength = unsafe Int(_swift_stdlib_strlen(leafPtr)) + // root + "." + leaf + appendedKVCLength = rootKVCLength &+ 1 &+ leafKVCLength &+ 1 + } else { + rootKVCLength = 0 + leafKVCLength = 0 + appendedKVCLength = 0 + } + + // Immediately following is the tail-allocated space for the KVC string. + let totalResultSize = MemoryLayout._roundingUpToAlignment(result &+ appendedKVCLength) + + return (result, totalResultSize, componentSize, appendedKVCLength, + rootKVCLength, leafKVCLength) +} + @usableFromInline @_unavailableInEmbedded internal func _appendingKeyPaths< @@ -2634,44 +2984,19 @@ internal func _appendingKeyPaths< return unsafe unsafeDowncast(leaf, to: Result.self) } - // Reserve room for the appended KVC string, if both key paths are - // KVC-compatible. - let appendedKVCLength: Int, rootKVCLength: Int, leafKVCLength: Int - - if root.getOffsetFromStorage() == nil, leaf.getOffsetFromStorage() == nil, - let rootPtr = unsafe root._kvcKeyPathStringPtr, - let leafPtr = unsafe leaf._kvcKeyPathStringPtr { - rootKVCLength = unsafe Int(_swift_stdlib_strlen(rootPtr)) - leafKVCLength = unsafe Int(_swift_stdlib_strlen(leafPtr)) - // root + "." + leaf - appendedKVCLength = rootKVCLength + 1 + leafKVCLength + 1 - } else { - rootKVCLength = 0 - leafKVCLength = 0 - appendedKVCLength = 0 - } - - // Result buffer has room for both key paths' components, plus the - // header, plus space for the middle type. - // Align up the root so that we can put the component type after it. - let rootSize = unsafe MemoryLayout._roundingUpToAlignment(rootBuffer.data.count) - var resultSize = unsafe rootSize + // Root component size - leafBuffer.data.count + // Leaf component size - MemoryLayout.size // Middle type - - // Size of just our components is equal to root + leaf + middle - let componentSize = resultSize - - resultSize += MemoryLayout.size // Header size (padding if needed) - - // The first member after the components is the maxSize of the keypath. - resultSize = MemoryLayout._roundingUpToAlignment(resultSize) - resultSize += MemoryLayout.size - - // Immediately following is the tail-allocated space for the KVC string. - let totalResultSize = MemoryLayout - ._roundingUpToAlignment(resultSize + appendedKVCLength) - + let ( + resultSize, + totalResultSize, + componentSize, + appendedKVCLength, + rootKVCLength, + leafKVCLength + ) = unsafe calculateAppendedKeyPathSize( + root, + leaf, + rootBuffer, + leafBuffer + ) var kvcStringBuffer: UnsafeMutableRawPointer? = nil let result = unsafe resultTy._create(capacityInBytes: totalResultSize) { @@ -2722,7 +3047,12 @@ internal func _appendingKeyPaths< unsafe component.clone( into: &destBuilder.buffer, - endOfReferencePrefix: endOfReferencePrefix) + endOfReferencePrefix: endOfReferencePrefix, + + // Root components don't need to adjust for alignment because the + // components should appear at the same offset as the root keypath. + adjustForAlignment: false + ) // Insert our endpoint type between the root and leaf components. if let type = type { unsafe destBuilder.push(type) @@ -2740,7 +3070,13 @@ internal func _appendingKeyPaths< unsafe component.clone( into: &destBuilder.buffer, - endOfReferencePrefix: component.header.endOfReferencePrefix) + endOfReferencePrefix: component.header.endOfReferencePrefix, + + // Leaf components, however, can appear at actual aligned spots that + // they couldn't before. Adjust computed arguments for this + // potential. + adjustForAlignment: true + ) if let type = type { unsafe destBuilder.push(type) @@ -2869,7 +3205,7 @@ public func _swift_getKeyPath(pattern: UnsafeMutableRawPointer, // Instantiate a new key path object modeled on the pattern. // Do a pass to determine the class of the key path we'll be instantiating // and how much space we'll need for it. - let (keyPathClass, rootType, size, sizeWithMaxSize, _) + let (keyPathClass, rootType, size, sizeWithMaxSize) = unsafe _getKeyPathClassAndInstanceSizeFromPattern(patternPtr, arguments) var pureStructOffset: UInt32? = nil @@ -3176,7 +3512,7 @@ internal func _walkKeyPathPattern( default: unsafe offset = unsafe .inline(header.storedOffsetPayload) } - let kind: KeyPathStructOrClass = header.kind == .struct + let kind: KeyPathStructOrClass = header.kind == .struct ? .struct : .class unsafe walker.visitStoredComponent(kind: kind, mutable: header.isStoredMutable, @@ -3431,6 +3767,9 @@ internal struct GetKeyPathClassAndInstanceSizeFromPattern // start with one word for the header var size: Int = MemoryLayout.size var sizeWithMaxSize: Int = 0 + var sizeWithObjectHeaderAndKvc: Int { + unsafe size &+ MemoryLayout.size &* 3 + } var capability: KeyPathKind = .value var didChain: Bool = false @@ -3527,46 +3866,50 @@ internal struct GetKeyPathClassAndInstanceSizeFromPattern if settable { unsafe size += MemoryLayout.size } - - // ...and the arguments, if any. - let argumentHeaderSize = MemoryLayout.size * 2 - switch unsafe (arguments, externalArgs) { - case (nil, nil): - break - case (let arguments?, nil): - unsafe size += argumentHeaderSize - // If we have arguments, calculate how much space they need by invoking - // the layout function. - let (addedSize, addedAlignmentMask) = unsafe arguments.getLayout(patternArgs) - // TODO: Handle over-aligned values - _internalInvariant(addedAlignmentMask < MemoryLayout.alignment, - "overaligned computed property element not supported") - unsafe size += addedSize - - case (let arguments?, let externalArgs?): - // If we're referencing an external declaration, and it takes captured - // arguments, then we have to build a bit of a chimera. The canonical - // identity and accessors come from the descriptor, but the argument - // handling is still as described in the local candidate. - unsafe size += argumentHeaderSize - let (addedSize, addedAlignmentMask) = unsafe arguments.getLayout(patternArgs) - // TODO: Handle over-aligned values - _internalInvariant(addedAlignmentMask < MemoryLayout.alignment, - "overaligned computed property element not supported") - unsafe size += addedSize - // We also need to store the size of the local arguments so we can - // find the external component arguments. + + // Handle local arguments. + if let arguments = unsafe arguments { + // Argument size and witnesses ptr. + unsafe size &+= MemoryLayout.size &* 2 + + // If we also have external arguments, we need to store the size of the + // local arguments so we can find the external component arguments. + if unsafe externalArgs != nil { + unsafe size &+= RawKeyPathComponent.Header.externalWithArgumentsExtraSize + } + + let (typeSize, typeAlignMask) = unsafe arguments.getLayout(patternArgs) + + // We are known to be pointer aligned at this point in the KeyPath buffer. + // However, for types who have an alignment large than pointers, we need + // to determine if the current position is suitable for the argument, or + // if we need to add padding bytes to align ourselves. + // + // Note: We need to account for the object header and kvc pointer because + // it's an odd number of words which will affect whether the size visitor + // determines if we need padding. It would cause out of sync answers when + // going to actually instantiate the buffer. + let misaligned = unsafe sizeWithObjectHeaderAndKvc & typeAlignMask + + if misaligned != 0 { + // We were misaligned, add the padding to the total size of the keypath + // buffer. + let typeAlign = typeAlignMask &+ 1 + unsafe size &+= typeAlign &- misaligned + } + + unsafe size &+= typeSize unsafe roundUpToPointerAlignment() - unsafe size += RawKeyPathComponent.Header.externalWithArgumentsExtraSize - unsafe size += MemoryLayout.size * externalArgs.count + } - case (nil, let externalArgs?): - // If we're instantiating an external property with a local - // candidate that has no arguments, then things are a little - // easier. We only need to instantiate the generic - // arguments for the external component's accessors. - unsafe size += argumentHeaderSize - unsafe size += MemoryLayout.size * externalArgs.count + // Handle external arguments. + if let externalArgs = unsafe externalArgs { + // Argument size and witnesses ptr if we didn't have local arguments. + if unsafe arguments == nil { + unsafe size &+= MemoryLayout.size &* 2 + } + + unsafe size &+= MemoryLayout.size &* externalArgs.count } } @@ -3599,8 +3942,7 @@ internal struct GetKeyPathClassAndInstanceSizeFromPattern } mutating func finish() { - unsafe sizeWithMaxSize = unsafe size - unsafe sizeWithMaxSize = unsafe MemoryLayout._roundingUpToAlignment(sizeWithMaxSize) + unsafe sizeWithMaxSize = unsafe MemoryLayout._roundingUpToAlignment(size) unsafe sizeWithMaxSize &+= MemoryLayout.size } } @@ -3613,8 +3955,7 @@ internal func _getKeyPathClassAndInstanceSizeFromPattern( keyPathClass: AnyKeyPath.Type, rootType: Any.Type, size: Int, - sizeWithMaxSize: Int, - alignmentMask: Int + sizeWithMaxSize: Int ) { var walker = unsafe GetKeyPathClassAndInstanceSizeFromPattern(patternArgs: arguments) unsafe _walkKeyPathPattern(pattern, walker: &walker) @@ -3640,12 +3981,12 @@ internal func _getKeyPathClassAndInstanceSizeFromPattern( } let classTy = unsafe _openExistential(walker.root!, do: openRoot) - return unsafe (keyPathClass: classTy, - rootType: walker.root!, - size: walker.size, - sizeWithMaxSize: walker.sizeWithMaxSize, - // FIXME: Handle overalignment - alignmentMask: MemoryLayout._alignmentMask) + return unsafe ( + keyPathClass: classTy, + rootType: walker.root!, + size: walker.size, + sizeWithMaxSize: walker.sizeWithMaxSize + ) } internal func _getTypeSize(_: Type.Type) -> Int { @@ -3891,22 +4232,12 @@ internal struct InstantiateKeyPathBuffer: KeyPathPatternVisitor { } if let arguments = unsafe arguments { - // Instantiate the arguments. - let (baseSize, alignmentMask) = unsafe arguments.getLayout(patternArgs) - _internalInvariant(alignmentMask < MemoryLayout.alignment, - "overaligned computed arguments not implemented yet") - - // The real buffer stride will be rounded up to alignment. - var totalSize = (baseSize + alignmentMask) & ~alignmentMask + // Record a placeholder for the total size of the arguments. We need to + // check after pushing preliminary data if the resulting argument will + // require padding bytes to be properly aligned. + let totalSizeAddress = unsafe destData.baseAddress._unsafelyUnwrappedUnchecked + unsafe pushDest(ComputedArgumentSize(0)) - // If an external property descriptor also has arguments, they'll be - // added to the end with pointer alignment. - if let externalArgs = unsafe externalArgs { - totalSize = MemoryLayout._roundingUpToAlignment(totalSize) - totalSize += MemoryLayout.size * externalArgs.count - } - - unsafe pushDest(totalSize) unsafe pushDest(arguments.witnesses) // A nonnull destructor in the witnesses file indicates the instantiated @@ -3922,15 +4253,60 @@ internal struct InstantiateKeyPathBuffer: KeyPathPatternVisitor { unsafe pushDest(externalArgs.count * MemoryLayout.size) } + let (typeSize, typeAlignMask) = unsafe arguments.getLayout(patternArgs) + var argumentSize = typeSize + + // If an external property descriptor also has arguments, they'll be + // added to the end with pointer alignment. + if let externalArgs = unsafe externalArgs { + argumentSize = MemoryLayout._roundingUpToAlignment(argumentSize) + argumentSize += MemoryLayout.size * externalArgs.count + } + + // The argument total size contains the padding and alignment bits + // required for the argument in the top 1/3 bits, so ensure the size of + // the argument buffer is small enough to account for that. + _precondition( + argumentSize <= ComputedArgumentSize.sizeMask, + "keypath arguments are too large" + ) + + var totalSize = ComputedArgumentSize(argumentSize) + let typeAlignment = typeAlignMask &+ 1 + totalSize.alignment = typeAlignment + + // We are known to be pointer aligned at this point in the KeyPath buffer. + // However, for types who have an alignment large than pointers, we need + // to determine if the current position is suitable for the argument, or + // if we need to add padding bytes to align ourselves. + let argumentAddress = unsafe destData.baseAddress._unsafelyUnwrappedUnchecked + let misaligned = Int(bitPattern: argumentAddress) & typeAlignMask + + if misaligned != 0 { + totalSize.padding = typeAlignment &- misaligned + + // Go ahead and append the padding. + for _ in 0 ..< totalSize.padding { + unsafe pushDest(UInt8(0)) + } + } + + // Ok, we've fully calculated totalSize, go ahead and update the + // placeholder. + unsafe totalSizeAddress.storeBytes( + of: totalSize, + as: ComputedArgumentSize.self + ) + // Initialize the local candidate arguments here. - unsafe _internalInvariant(Int(bitPattern: destData.baseAddress) & alignmentMask == 0, + unsafe _internalInvariant(Int(bitPattern: destData.baseAddress) & typeAlignMask == 0, "argument destination not aligned") unsafe arguments.initializer(patternArgs, destData.baseAddress._unsafelyUnwrappedUnchecked) unsafe destData = unsafe UnsafeMutableRawBufferPointer( - start: destData.baseAddress._unsafelyUnwrappedUnchecked + baseSize, - count: destData.count - baseSize) + start: destData.baseAddress._unsafelyUnwrappedUnchecked + typeSize, + count: destData.count - typeSize) } if let externalArgs = unsafe externalArgs { @@ -4044,7 +4420,6 @@ internal struct ValidatingInstantiateKeyPathBuffer: KeyPathPatternVisitor { offset: offset) unsafe checkSizeConsistency() unsafe structOffset = unsafe instantiateVisitor.structOffset - unsafe isPureStruct.append(contentsOf: instantiateVisitor.isPureStruct) } mutating func visitComputedComponent(mutating: Bool, idKind: KeyPathComputedIDKind, @@ -4076,31 +4451,26 @@ internal struct ValidatingInstantiateKeyPathBuffer: KeyPathPatternVisitor { // Note: For this function and the ones below, modification of structOffset // is omitted since these types of KeyPaths won't have a pureStruct // offset anyway. - unsafe isPureStruct.append(contentsOf: instantiateVisitor.isPureStruct) unsafe checkSizeConsistency() } mutating func visitOptionalChainComponent() { unsafe sizeVisitor.visitOptionalChainComponent() unsafe instantiateVisitor.visitOptionalChainComponent() - unsafe isPureStruct.append(contentsOf: instantiateVisitor.isPureStruct) unsafe checkSizeConsistency() } mutating func visitOptionalWrapComponent() { unsafe sizeVisitor.visitOptionalWrapComponent() unsafe instantiateVisitor.visitOptionalWrapComponent() - unsafe isPureStruct.append(contentsOf: instantiateVisitor.isPureStruct) unsafe checkSizeConsistency() } mutating func visitOptionalForceComponent() { unsafe sizeVisitor.visitOptionalForceComponent() unsafe instantiateVisitor.visitOptionalForceComponent() - unsafe isPureStruct.append(contentsOf: instantiateVisitor.isPureStruct) unsafe checkSizeConsistency() } mutating func visitIntermediateComponentType(metadataRef: MetadataReference) { unsafe sizeVisitor.visitIntermediateComponentType(metadataRef: metadataRef) unsafe instantiateVisitor.visitIntermediateComponentType(metadataRef: metadataRef) - unsafe isPureStruct.append(contentsOf: instantiateVisitor.isPureStruct) unsafe checkSizeConsistency() } @@ -4245,7 +4615,13 @@ public func _createOffsetBasedKeyPath( body: UnsafeRawBufferPointer(start: nil, count: 0) ) - unsafe component.clone(into: &builder.buffer, endOfReferencePrefix: false) + unsafe component.clone( + into: &builder.buffer, + endOfReferencePrefix: false, + + // The keypath will have the same shape, it's just the types that differ. + adjustForAlignment: false + ) } if _MetadataKind(root) == .struct { @@ -4319,7 +4695,11 @@ public func _rerootKeyPath( unsafe rawComponent.clone( into: &builder.buffer, - endOfReferencePrefix: rawComponent.header.endOfReferencePrefix + endOfReferencePrefix: rawComponent.header.endOfReferencePrefix, + + // Simply changing the type of the root type, so the resulting buffer + // will have the same layout. + adjustForAlignment: false ) if componentTy == nil { diff --git a/stdlib/public/core/ReflectionMirror.swift b/stdlib/public/core/ReflectionMirror.swift index 73fa13048c0d9..3a32b38a6a6c8 100644 --- a/stdlib/public/core/ReflectionMirror.swift +++ b/stdlib/public/core/ReflectionMirror.swift @@ -378,7 +378,11 @@ public func _forEachFieldWithKeyPath( body: UnsafeRawBufferPointer(start: nil, count: 0)) unsafe component.clone( into: &destBuilder.buffer, - endOfReferencePrefix: false) + endOfReferencePrefix: false, + + // We are just storing offset components and not computed ones. + adjustForAlignment: false + ) } if let name = unsafe field.name { diff --git a/test/stdlib/KeyPath.swift b/test/stdlib/KeyPath.swift index fb3b0c1e5510c..5da22242f33b7 100644 --- a/test/stdlib/KeyPath.swift +++ b/test/stdlib/KeyPath.swift @@ -1159,5 +1159,117 @@ if #available(SwiftStdlib 5.9, *) { } } +@_alignment(8) +struct EightAligned { + let x = 0 +} + +extension EightAligned: Equatable {} +extension EightAligned: Hashable {} + +@_alignment(16) +struct SixteenAligned { + let x = 0 +} + +extension SixteenAligned: Equatable {} +extension SixteenAligned: Hashable {} + +struct OveralignedForEight { + subscript(getter getter: EightAligned) -> Int { + 128 + } + + subscript(setter setter: EightAligned) -> Int { + get { + 128 + } + + set { + fatalError() + } + } +} + +struct OveralignedForSixteen { + subscript(getter getter: SixteenAligned) -> Int { + 128 + } + + subscript(setter setter: SixteenAligned) -> Int { + get { + 128 + } + + set { + fatalError() + } + } +} + +struct SimpleEight { + let oa = OveralignedForEight() +} + +struct SimpleSixteen { + let oa = OveralignedForSixteen() +} + +#if _pointerBitWidth(_32) +if #available(SwiftStdlib 6.3, *) { + keyPath.test("eight byte overaligned arguments") { + let oa = OveralignedForEight() + let kp = \OveralignedForEight.[getter: EightAligned()] + + let value = oa[keyPath: kp] + expectEqual(value, 128) + + // Test that appends where the argument is no longer overaligned work + let simple = SimpleEight() + let kp11 = \SimpleEight.oa + let kp12 = \OveralignedForEight.[getter: EightAligned()] + let kp1 = kp11.appending(path: kp12) + + let value1 = simple[keyPath: kp1] + expectEqual(value1, 128) + + // Test that appends where the argument is still overaligned works + let kp21 = \SimpleEight.oa + let kp22 = \OveralignedForEight.[setter: EightAligned()] + let kp2 = kp21.appending(path: kp22) + + let value2 = simple[keyPath: kp2] + expectEqual(value2, 128) + } +} +#endif + +if #available(SwiftStdlib 6.3, *) { + keyPath.test("sixteen byte overaligned arguments") { + let oa = OveralignedForSixteen() + let kp = \OveralignedForSixteen.[getter: SixteenAligned()] + + let value = oa[keyPath: kp] + expectEqual(value, 128) + + // Test that appends where the argument is no longer overaligned work + let simple = SimpleSixteen() + let kp11 = \SimpleSixteen.oa + let kp12 = \OveralignedForSixteen.[getter: SixteenAligned()] + let kp1 = kp11.appending(path: kp12) + + let value1 = simple[keyPath: kp1] + expectEqual(value1, 128) + + // Test that appends where the argument is still overaligned works + let kp21 = \SimpleSixteen.oa + let kp22 = \OveralignedForSixteen.[setter: SixteenAligned()] + let kp2 = kp21.appending(path: kp22) + + let value2 = simple[keyPath: kp2] + expectEqual(value2, 128) + } +} + runAllTests()