Skip to content

Commit

Permalink
Merge pull request #25418 from Catfish-Man/no-objc-complications-4
Browse files Browse the repository at this point in the history 
Use the remaining half bit in the refcount to bypass ObjC deallocation overhead
  • Loading branch information
@Catfish-Man
Catfish-Man committed Jun 19, 2019
2 parents 379d88c + c512946 commit 28dcc91
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 45 deletions.
135 changes: 99 additions & 36 deletions stdlib/public/SwiftShims/RefCount.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,14 +238,29 @@ struct RefCountBitOffsets;
// 32-bit out of line
template <>
struct RefCountBitOffsets<8> {
static const size_t IsImmortalShift = 0;
static const size_t IsImmortalBitCount = 1;
static const uint64_t IsImmortalMask = maskForField(IsImmortal);

static const size_t UnownedRefCountShift = shiftAfterField(IsImmortal);
/*
The bottom 32 bits (on 64 bit architectures, fewer on 32 bit) of the refcount
field are effectively a union of two different configurations:
---Normal case---
Bit 0: Does this object need to call out to the ObjC runtime for deallocation
Bits 1-31: Unowned refcount
---Immortal case---
All bits set, the object does not deallocate or have a refcount
*/
static const size_t PureSwiftDeallocShift = 0;
static const size_t PureSwiftDeallocBitCount = 1;
static const uint64_t PureSwiftDeallocMask = maskForField(PureSwiftDealloc);

static const size_t UnownedRefCountShift = shiftAfterField(PureSwiftDealloc);
static const size_t UnownedRefCountBitCount = 31;
static const uint64_t UnownedRefCountMask = maskForField(UnownedRefCount);

static const size_t IsImmortalShift = 0; // overlaps PureSwiftDealloc and UnownedRefCount
static const size_t IsImmortalBitCount = 32;
static const uint64_t IsImmortalMask = maskForField(IsImmortal);

static const size_t IsDeinitingShift = shiftAfterField(UnownedRefCount);
static const size_t IsDeinitingBitCount = 1;
static const uint64_t IsDeinitingMask = maskForField(IsDeiniting);
Expand All @@ -271,14 +286,18 @@ struct RefCountBitOffsets<8> {
// 32-bit inline
template <>
struct RefCountBitOffsets<4> {
static const size_t IsImmortalShift = 0;
static const size_t IsImmortalBitCount = 1;
static const uint64_t IsImmortalMask = maskForField(IsImmortal);
static const size_t PureSwiftDeallocShift = 0;
static const size_t PureSwiftDeallocBitCount = 1;
static const uint32_t PureSwiftDeallocMask = maskForField(PureSwiftDealloc);

static const size_t UnownedRefCountShift = shiftAfterField(IsImmortal);
static const size_t UnownedRefCountShift = shiftAfterField(PureSwiftDealloc);
static const size_t UnownedRefCountBitCount = 7;
static const uint32_t UnownedRefCountMask = maskForField(UnownedRefCount);

static const size_t IsImmortalShift = 0; // overlaps PureSwiftDealloc and UnownedRefCount
static const size_t IsImmortalBitCount = 8;
static const uint32_t IsImmortalMask = maskForField(IsImmortal);

static const size_t IsDeinitingShift = shiftAfterField(UnownedRefCount);
static const size_t IsDeinitingBitCount = 1;
static const uint32_t IsDeinitingMask = maskForField(IsDeiniting);
Expand Down Expand Up @@ -369,33 +388,56 @@ class RefCountBitsT {
enum Immortal_t { Immortal };

LLVM_ATTRIBUTE_ALWAYS_INLINE
bool isImmortal() const {
return bool(getField(IsImmortal));
bool isImmortal(bool checkSlowRCBit) const {
if (checkSlowRCBit) {
return (getField(IsImmortal) == Offsets::IsImmortalMask) &&
bool(getField(UseSlowRC));
} else {
return (getField(IsImmortal) == Offsets::IsImmortalMask);
}
}

LLVM_ATTRIBUTE_ALWAYS_INLINE
bool isOverflowingUnownedRefCount(uint32_t oldValue, uint32_t inc) const {
auto newValue = getUnownedRefCount();
return newValue != oldValue + inc ||
newValue == Offsets::UnownedRefCountMask;
}

LLVM_ATTRIBUTE_ALWAYS_INLINE
void setIsImmortal(bool value) {
setField(IsImmortal, value);
assert(value);
setField(IsImmortal, Offsets::IsImmortalMask);
setField(UseSlowRC, value);
}

LLVM_ATTRIBUTE_ALWAYS_INLINE
bool pureSwiftDeallocation() const {
return bool(getField(PureSwiftDealloc)) && !bool(getField(UseSlowRC));
}

LLVM_ATTRIBUTE_ALWAYS_INLINE
void setPureSwiftDeallocation(bool value) {
setField(PureSwiftDealloc, value);
}

LLVM_ATTRIBUTE_ALWAYS_INLINE
RefCountBitsT() = default;

LLVM_ATTRIBUTE_ALWAYS_INLINE
constexpr
RefCountBitsT(uint32_t strongExtraCount, uint32_t unownedCount)
: bits((BitsType(strongExtraCount) << Offsets::StrongExtraRefCountShift) |
(BitsType(1) << Offsets::PureSwiftDeallocShift) |
(BitsType(unownedCount) << Offsets::UnownedRefCountShift))
{ }

LLVM_ATTRIBUTE_ALWAYS_INLINE
constexpr
RefCountBitsT(Immortal_t immortal)
: bits((BitsType(2) << Offsets::StrongExtraRefCountShift) |
(BitsType(2) << Offsets::UnownedRefCountShift) |
(BitsType(1) << Offsets::IsImmortalShift) |
(BitsType(1) << Offsets::UseSlowRCShift))
: bits((BitsType(2) << Offsets::StrongExtraRefCountShift) |
(BitsType(Offsets::IsImmortalMask)) |
(BitsType(1) << Offsets::UseSlowRCShift))
{ }

LLVM_ATTRIBUTE_ALWAYS_INLINE
Expand Down Expand Up @@ -433,7 +475,7 @@ class RefCountBitsT {

LLVM_ATTRIBUTE_ALWAYS_INLINE
bool hasSideTable() const {
bool hasSide = getUseSlowRC() && !isImmortal();
bool hasSide = getUseSlowRC() && !isImmortal(false);

// Side table refcount must not point to another side table.
assert((refcountIsInline || !hasSide) &&
Expand Down Expand Up @@ -523,7 +565,7 @@ class RefCountBitsT {
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE
bool decrementStrongExtraRefCount(uint32_t dec) {
#ifndef NDEBUG
if (!hasSideTable() && !isImmortal()) {
if (!hasSideTable() && !isImmortal(false)) {
// Can't check these assertions with side table present.

if (getIsDeiniting())
Expand Down Expand Up @@ -558,7 +600,7 @@ class RefCountBitsT {
static_assert(Offsets::UnownedRefCountBitCount +
Offsets::IsDeinitingBitCount +
Offsets::StrongExtraRefCountBitCount +
Offsets::IsImmortalBitCount +
Offsets::PureSwiftDeallocBitCount +
Offsets::UseSlowRCBitCount == sizeof(bits)*8,
"inspect isUniquelyReferenced after adding fields");

Expand Down Expand Up @@ -715,7 +757,7 @@ class RefCounts {

void setIsImmortal(bool immortal) {
auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
if (oldbits.isImmortal()) {
if (oldbits.isImmortal(true)) {
return;
}
RefCountBits newbits;
Expand All @@ -725,7 +767,28 @@ class RefCounts {
} while (!refCounts.compare_exchange_weak(oldbits, newbits,
std::memory_order_relaxed));
}


void setPureSwiftDeallocation(bool nonobjc) {
auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
//Immortal and no objc complications share a bit, so don't let setting
//the complications one clear the immmortal one
if (oldbits.isImmortal(true) || oldbits.pureSwiftDeallocation() == nonobjc){
assert(!oldbits.hasSideTable());
return;
}
RefCountBits newbits;
do {
newbits = oldbits;
newbits.setPureSwiftDeallocation(nonobjc);
} while (!refCounts.compare_exchange_weak(oldbits, newbits,
std::memory_order_relaxed));
}

bool getPureSwiftDeallocation() {
auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
return bits.pureSwiftDeallocation();
}

// Initialize from another refcount bits.
// Only inline -> out-of-line is allowed (used for new side table entries).
void init(InlineRefCountBits newBits) {
Expand All @@ -740,7 +803,7 @@ class RefCounts {
newbits = oldbits;
bool fast = newbits.incrementStrongExtraRefCount(inc);
if (SWIFT_UNLIKELY(!fast)) {
if (oldbits.isImmortal())
if (oldbits.isImmortal(false))
return;
return incrementSlow(oldbits, inc);
}
Expand All @@ -753,7 +816,7 @@ class RefCounts {
auto newbits = oldbits;
bool fast = newbits.incrementStrongExtraRefCount(inc);
if (SWIFT_UNLIKELY(!fast)) {
if (oldbits.isImmortal())
if (oldbits.isImmortal(false))
return;
return incrementNonAtomicSlow(oldbits, inc);
}
Expand All @@ -771,7 +834,7 @@ class RefCounts {
newbits = oldbits;
bool fast = newbits.incrementStrongExtraRefCount(1);
if (SWIFT_UNLIKELY(!fast)) {
if (oldbits.isImmortal())
if (oldbits.isImmortal(false))
return true;
return tryIncrementSlow(oldbits);
}
Expand All @@ -788,7 +851,7 @@ class RefCounts {
auto newbits = oldbits;
bool fast = newbits.incrementStrongExtraRefCount(1);
if (SWIFT_UNLIKELY(!fast)) {
if (oldbits.isImmortal())
if (oldbits.isImmortal(false))
return true;
return tryIncrementNonAtomicSlow(oldbits);
}
Expand Down Expand Up @@ -824,7 +887,7 @@ class RefCounts {
// Precondition: the reference count must be 1
void decrementFromOneNonAtomic() {
auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
if (bits.isImmortal()) {
if (bits.isImmortal(true)) {
return;
}
if (bits.hasSideTable())
Expand Down Expand Up @@ -922,7 +985,7 @@ class RefCounts {
// Decrement completed normally. New refcount is not zero.
deinitNow = false;
}
else if (oldbits.isImmortal()) {
else if (oldbits.isImmortal(false)) {
return false;
} else if (oldbits.hasSideTable()) {
// Decrement failed because we're on some other slow path.
Expand Down Expand Up @@ -961,7 +1024,7 @@ class RefCounts {
// Decrement completed normally. New refcount is not zero.
deinitNow = false;
}
else if (oldbits.isImmortal()) {
else if (oldbits.isImmortal(false)) {
return false;
}
else if (oldbits.hasSideTable()) {
Expand Down Expand Up @@ -1001,7 +1064,7 @@ class RefCounts {
bool fast =
newbits.decrementStrongExtraRefCount(dec);
if (SWIFT_UNLIKELY(!fast)) {
if (oldbits.isImmortal()) {
if (oldbits.isImmortal(false)) {
return false;
}
// Slow paths include side table; deinit; underflow
Expand All @@ -1025,7 +1088,7 @@ class RefCounts {
// Increment the unowned reference count.
void incrementUnowned(uint32_t inc) {
auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
if (oldbits.isImmortal())
if (oldbits.isImmortal(true))
return;
RefCountBits newbits;
do {
Expand All @@ -1037,7 +1100,7 @@ class RefCounts {
uint32_t oldValue = newbits.incrementUnownedRefCount(inc);

// Check overflow and use the side table on overflow.
if (newbits.getUnownedRefCount() != oldValue + inc)
if (newbits.isOverflowingUnownedRefCount(oldValue, inc))
return incrementUnownedSlow(inc);

} while (!refCounts.compare_exchange_weak(oldbits, newbits,
Expand All @@ -1046,7 +1109,7 @@ class RefCounts {

void incrementUnownedNonAtomic(uint32_t inc) {
auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
if (oldbits.isImmortal())
if (oldbits.isImmortal(true))
return;
if (oldbits.hasSideTable())
return oldbits.getSideTable()->incrementUnownedNonAtomic(inc);
Expand All @@ -1056,7 +1119,7 @@ class RefCounts {
uint32_t oldValue = newbits.incrementUnownedRefCount(inc);

// Check overflow and use the side table on overflow.
if (newbits.getUnownedRefCount() != oldValue + inc)
if (newbits.isOverflowingUnownedRefCount(oldValue, inc))
return incrementUnownedSlow(inc);

refCounts.store(newbits, std::memory_order_relaxed);
Expand All @@ -1066,7 +1129,7 @@ class RefCounts {
// Return true if the caller should free the object.
bool decrementUnownedShouldFree(uint32_t dec) {
auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
if (oldbits.isImmortal())
if (oldbits.isImmortal(true))
return false;
RefCountBits newbits;

Expand Down Expand Up @@ -1094,7 +1157,7 @@ class RefCounts {

bool decrementUnownedShouldFreeNonAtomic(uint32_t dec) {
auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
if (oldbits.isImmortal())
if (oldbits.isImmortal(true))
return false;
if (oldbits.hasSideTable())
return oldbits.getSideTable()->decrementUnownedShouldFreeNonAtomic(dec);
Expand Down Expand Up @@ -1383,7 +1446,7 @@ inline bool RefCounts<InlineRefCountBits>::doDecrementNonAtomic(uint32_t dec) {
auto newbits = oldbits;
bool fast = newbits.decrementStrongExtraRefCount(dec);
if (!fast) {
if (oldbits.isImmortal()) {
if (oldbits.isImmortal(false)) {
return false;
}
return doDecrementNonAtomicSlow<performDeinit>(oldbits, dec);
Expand Down
41 changes: 40 additions & 1 deletion stdlib/public/runtime/HeapObject.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
# include <objc/message.h>
# include <objc/objc.h>
# include "swift/Runtime/ObjCBridge.h"
# include "swift/Runtime/Once.h"
#endif
#include "Leaks.h"

Expand Down Expand Up @@ -78,6 +79,34 @@ HeapObject *swift::swift_allocObject(HeapMetadata const *metadata,
return _swift_allocObject(metadata, requiredSize, requiredAlignmentMask);
}

#if OBJC_SETASSOCIATEDOBJECTHOOK_DEFINED
//We interpose objc_setAssociatedObject so that we can set a flag in
//the refcount field of Swift objects to indicate that they have associations,
//since we can't safely skip ObjC dealloc work if they do
static objc_hook_setAssociatedObject originalAssocObjectFunc = nullptr;

static void _swift_setAssociatedObject_hook(
id _Nonnull object,
const void * _Nonnull key,
id _Nullable value,
objc_AssociationPolicy policy
) {
if (!isObjCTaggedPointerOrNull(object) &&
objectUsesNativeSwiftReferenceCounting(object)) {
auto heapObj = reinterpret_cast<HeapObject *>(object);
heapObj->refCounts.setPureSwiftDeallocation(false);
}
originalAssocObjectFunc(object, key, value, policy);
}

static void _interpose_objc_association(void *ctxt) {
if (__builtin_available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *)) {
objc_setHook_setAssociatedObject(_swift_setAssociatedObject_hook,
&originalAssocObjectFunc);
}
}
#endif

static HeapObject *_swift_allocObject_(HeapMetadata const *metadata,
size_t requiredSize,
size_t requiredAlignmentMask) {
Expand All @@ -90,6 +119,11 @@ static HeapObject *_swift_allocObject_(HeapMetadata const *metadata,
// Linux, and macOS.
new (object) HeapObject(metadata);

#if OBJC_SETASSOCIATEDOBJECTHOOK_DEFINED
static swift_once_t associatedObjectHookOnce;
swift_once(&associatedObjectHookOnce, _interpose_objc_association, nullptr);
#endif

// If leak tracking is enabled, start tracking this object.
SWIFT_LEAKS_START_TRACKING_OBJECT(object);

Expand Down Expand Up @@ -594,9 +628,14 @@ void swift::swift_rootObjCDealloc(HeapObject *self) {
void swift::swift_deallocClassInstance(HeapObject *object,
size_t allocatedSize,
size_t allocatedAlignMask) {
#if SWIFT_OBJC_INTEROP
#if OBJC_SETASSOCIATEDOBJECTHOOK_DEFINED
// We need to let the ObjC runtime clean up any associated objects or weak
// references associated with this object.
if (originalAssocObjectFunc == nullptr ||
!object->refCounts.getPureSwiftDeallocation()) {
objc_destructInstance((id)object);
}
#elif SWIFT_OBJC_INTEROP
objc_destructInstance((id)object);
#endif
swift_deallocObject(object, allocatedSize, allocatedAlignMask);
Expand Down
Loading

0 comments on commit 28dcc91

Please sign in to comment.