Introduce compile-time checking and alternate allocation for compact …

…pointer types

https://bugs.webkit.org/show_bug.cgi?id=266080
rdar://119382681

Reviewed by Mark Lam.

Adds compile-time checking to WTF compact and packed pointer types requiring the
pointed-to types to be explicitly marked as compatible with compact pointers. Also
adds new equivalents to the existing WTF::fastMalloc and WTF::isoMalloc allocation
functions for compactible types, giving us an API point to play with how types
used in compact pointers are allocated.

* Source/JavaScriptCore/bytecode/CallLinkInfoBase.h:
* Source/JavaScriptCore/bytecode/CodeBlock.h:
* Source/JavaScriptCore/bytecode/InlineCallFrame.h:
* Source/JavaScriptCore/bytecode/InlineCallFrameSet.h:
* Source/JavaScriptCore/bytecode/StructureStubClearingWatchpoint.h:
* Source/JavaScriptCore/bytecode/Watchpoint.h:
* Source/JavaScriptCore/dfg/DFGJITCode.h:
* Source/JavaScriptCore/heap/FastMallocAlignedMemoryAllocator.cpp:
* Source/JavaScriptCore/heap/IsoAlignedMemoryAllocator.cpp:
* Source/JavaScriptCore/jit/BaselineJITCode.h:
* Source/JavaScriptCore/jit/ExecutableMemoryHandle.h:
* Source/JavaScriptCore/jit/JITCode.h:
* Source/JavaScriptCore/jit/PolymorphicCallStubRoutine.h:
* Source/JavaScriptCore/runtime/ClassInfo.h:
* Source/JavaScriptCore/runtime/JSArray.h:
* Source/JavaScriptCore/wasm/WasmTypeDefinition.h:
* Source/WTF/wtf/Bag.h:
* Source/WTF/wtf/CompactPointerTuple.h:
* Source/WTF/wtf/CompactPtr.h:
* Source/WTF/wtf/DebugHeap.h:
* Source/WTF/wtf/FastMalloc.cpp:
* Source/WTF/wtf/FastMalloc.h:
* Source/WTF/wtf/IsoMalloc.h:
* Source/WTF/wtf/IsoMallocInlines.h:
* Source/WTF/wtf/SentinelLinkedList.h:
* Source/WTF/wtf/WeakPtr.h:
* Source/WTF/wtf/text/StringImpl.h:
* Source/WTF/wtf/text/WTFString.h:
* Source/WebCore/dom/Node.h:
* Source/WebCore/dom/NodeRareData.h:
* Source/WebCore/rendering/RenderObject.cpp:
* Source/WebCore/rendering/RenderObject.h:
* Source/bmalloc/bmalloc/AllocationCounts.h:
* Source/bmalloc/bmalloc/IsoHeap.cpp:
* Source/bmalloc/bmalloc/IsoHeap.h:
* Source/bmalloc/bmalloc/IsoHeapImpl.h:
* Source/bmalloc/bmalloc/IsoHeapInlines.h:
* Source/bmalloc/bmalloc/Packed.h:
* Tools/TestWebKitAPI/Tests/WTF/AlignedRefLogger.h:
* Tools/TestWebKitAPI/Tests/WTF/CompactRefPtr.cpp:
* Tools/TestWebKitAPI/Tests/WTF/CompactUniquePtrTuple.cpp:
* Tools/TestWebKitAPI/Tests/WTF/RefLogger.h:

Canonical link: https://commits.webkit.org/271796@main

Source/JavaScriptCore/bytecode/CallLinkInfoBase.h

@@ -31,7 +31,7 @@ namespace JSC {
 
 class VM;
 
-class CallLinkInfoBase : public PackedRawSentinelNode<CallLinkInfoBase> {
+class CallLinkInfoBase : public BasicRawSentinelNode<CallLinkInfoBase> {
 public:
     enum class CallSiteType : uint8_t {
         CallLinkInfo,

Source/JavaScriptCore/bytecode/CodeBlock.h

@@ -952,7 +952,7 @@ class CodeBlock : public JSCell {
     VM* const m_vm;
 
     const void* const m_instructionsRawPointer { nullptr };
-    SentinelLinkedList<CallLinkInfoBase, PackedRawSentinelNode<CallLinkInfoBase>> m_incomingCalls;
+    SentinelLinkedList<CallLinkInfoBase, BasicRawSentinelNode<CallLinkInfoBase>> m_incomingCalls;
     uint16_t m_optimizationDelayCounter { 0 };
     uint16_t m_reoptimizationRetryCounter { 0 };
     StructureWatchpointMap m_llintGetByIdWatchpointMap;

Source/JavaScriptCore/bytecode/InlineCallFrame.h

@@ -40,7 +40,11 @@ struct InlineCallFrame;
 class CallFrame;
 class JSFunction;
 
+DECLARE_COMPACT_ALLOCATOR_WITH_HEAP_IDENTIFIER(InlineCallFrame);
+
 struct InlineCallFrame {
+    WTF_MAKE_STRUCT_FAST_COMPACT_ALLOCATED_WITH_HEAP_IDENTIFIER(InlineCallFrame);
+
     enum Kind {
         Call,
         Construct,

Source/JavaScriptCore/bytecode/InlineCallFrameSet.h

@@ -41,12 +41,13 @@ class InlineCallFrameSet : public RefCounted<InlineCallFrameSet> {
 
     InlineCallFrame* add();
 
-    typedef Bag<InlineCallFrame>::iterator iterator;
+    using FrameBag = Bag<InlineCallFrame, RawPtrTraits<InlineCallFrame>, InlineCallFrameMalloc>;
+    typedef FrameBag::iterator iterator;
     iterator begin() { return m_frames.begin(); }
     iterator end() { return m_frames.end(); }
 
 private:
-    Bag<InlineCallFrame> m_frames;
+    FrameBag m_frames;
 };
 
 } // namespace JSC

Source/JavaScriptCore/bytecode/StructureStubClearingWatchpoint.h

@@ -54,7 +54,7 @@ class StructureTransitionStructureStubClearingWatchpoint final : public Watchpoi
     void fireInternal(VM&, const FireDetail&);
 
 private:
-    PackedPtr<WatchpointsOnStructureStubInfo> m_holder;
+    WatchpointsOnStructureStubInfo* m_holder;
     ObjectPropertyCondition m_key;
 };
 
@@ -75,7 +75,7 @@ class AdaptiveValueStructureStubClearingWatchpoint final : public AdaptiveInferr
 
 
 private:
-    PackedPtr<WatchpointsOnStructureStubInfo> m_holder;
+    WatchpointsOnStructureStubInfo* m_holder;
 };
 
 class WatchpointsOnStructureStubInfo final {

Source/JavaScriptCore/bytecode/Watchpoint.h

@@ -111,7 +111,7 @@ class WatchpointSet;
 
 DECLARE_ALLOCATOR_WITH_HEAP_IDENTIFIER(Watchpoint);
 
-class Watchpoint : public PackedRawSentinelNode<Watchpoint> {
+class Watchpoint : public BasicRawSentinelNode<Watchpoint> {
     WTF_MAKE_NONCOPYABLE(Watchpoint);
     WTF_MAKE_NONMOVABLE(Watchpoint);
     WTF_MAKE_STRUCT_FAST_ALLOCATED_WITH_HEAP_IDENTIFIER(Watchpoint);
@@ -274,7 +274,7 @@ class WatchpointSet : public ThreadSafeRefCounted<WatchpointSet> {
     int8_t m_state;
     int8_t m_setIsNotEmpty;
 
-    SentinelLinkedList<Watchpoint, PackedRawSentinelNode<Watchpoint>> m_set;
+    SentinelLinkedList<Watchpoint, BasicRawSentinelNode<Watchpoint>> m_set;
 };
 
 // InlineWatchpointSet is a low-overhead, non-copyable watchpoint set in which

Source/JavaScriptCore/dfg/DFGJITCode.h

@@ -94,7 +94,7 @@ class LinkerIR {
 public:
     using Constant = unsigned;
 
-    enum class Type : uint16_t {
+    enum class Type : uint8_t {
         Invalid,
         CallLinkInfo,
         CellPointer,
@@ -115,12 +115,12 @@ class LinkerIR {
         ObjectPrototypeChainIsSaneWatchpointSet,
     };
 
-    using Value = CompactPointerTuple<void*, Type>;
+    using Value = JITConstant<Type>;
 
     struct ValueHash {
         static unsigned hash(const Value& p)
         {
-            return computeHash(p.type(), p.pointer());
+            return p.hash();
         }
 
         static bool equal(const Value& a, const Value& b)

Source/JavaScriptCore/heap/FastMallocAlignedMemoryAllocator.cpp

@@ -46,7 +46,7 @@ void* FastMallocAlignedMemoryAllocator::tryAllocateAlignedMemory(size_t alignmen
 #if ENABLE(MALLOC_HEAP_BREAKDOWN)
     return m_heap.memalign(alignment, size, true);
 #else
-    return tryFastAlignedMalloc(alignment, size);
+    return tryFastCompactAlignedMalloc(alignment, size);
 #endif
 
 }
@@ -71,7 +71,7 @@ void* FastMallocAlignedMemoryAllocator::tryAllocateMemory(size_t size)
 #if ENABLE(MALLOC_HEAP_BREAKDOWN)
     return m_heap.malloc(size);
 #else
-    return FastMalloc::tryMalloc(size);
+    return FastCompactMalloc::tryMalloc(size);
 #endif
 }
 
@@ -80,7 +80,7 @@ void FastMallocAlignedMemoryAllocator::freeMemory(void* pointer)
 #if ENABLE(MALLOC_HEAP_BREAKDOWN)
     return m_heap.free(pointer);
 #else
-    FastMalloc::free(pointer);
+    FastCompactMalloc::free(pointer);
 #endif
 }
 
@@ -89,7 +89,7 @@ void* FastMallocAlignedMemoryAllocator::tryReallocateMemory(void* pointer, size_
 #if ENABLE(MALLOC_HEAP_BREAKDOWN)
     return m_heap.realloc(pointer, size);
 #else
-    return FastMalloc::tryRealloc(pointer, size);
+    return FastCompactMalloc::tryRealloc(pointer, size);
 #endif
 }
 

Source/JavaScriptCore/heap/IsoAlignedMemoryAllocator.cpp

@@ -52,7 +52,7 @@ void* IsoAlignedMemoryAllocator::tryAllocateMemory(size_t size)
 #if ENABLE(MALLOC_HEAP_BREAKDOWN)
     return m_heap.malloc(size);
 #else
-    return FastMalloc::tryMalloc(size);
+    return FastCompactMalloc::tryMalloc(size);
 #endif
 }
 
@@ -76,7 +76,7 @@ void* IsoAlignedMemoryAllocator::tryMallocBlock()
 #if ENABLE(MALLOC_HEAP_BREAKDOWN)
     return m_heap.memalign(MarkedBlock::blockSize, MarkedBlock::blockSize, true);
 #else
-    return tryFastAlignedMalloc(MarkedBlock::blockSize, MarkedBlock::blockSize);
+    return tryFastCompactAlignedMalloc(MarkedBlock::blockSize, MarkedBlock::blockSize);
 #endif
 }
 

Source/JavaScriptCore/heap/IsoCellSet.cpp

@@ -42,7 +42,7 @@ IsoCellSet::IsoCellSet(IsoSubspace& subspace)
 IsoCellSet::~IsoCellSet()
 {
     if (isOnList())
-        PackedRawSentinelNode<IsoCellSet>::remove();
+        BasicRawSentinelNode<IsoCellSet>::remove();
 }
 
 Ref<SharedTask<MarkedBlock::Handle*()>> IsoCellSet::parallelNotEmptyMarkedBlockSource()

Source/JavaScriptCore/heap/IsoCellSet.h

@@ -42,7 +42,7 @@ class IsoSubspace;
 // Create a set of cells that are in an IsoSubspace. This allows concurrent O(1) set insertion and
 // removal. Each such set should be thought of as a 0.8% increase in object size for objects in that
 // IsoSubspace (it's like adding 1 bit every 16 bytes, or 1 bit every 128 bits).
-class IsoCellSet final : public PackedRawSentinelNode<IsoCellSet> {
+class IsoCellSet final : public BasicRawSentinelNode<IsoCellSet> {
     WTF_MAKE_NONCOPYABLE(IsoCellSet);
     WTF_MAKE_NONMOVABLE(IsoCellSet);
 public:

Source/JavaScriptCore/heap/IsoSubspace.h

@@ -66,8 +66,8 @@ class IsoSubspace : public Subspace {
 
     BlockDirectory m_directory;
     std::unique_ptr<IsoMemoryAllocatorBase> m_isoAlignedMemoryAllocator;
-    SentinelLinkedList<PreciseAllocation, PackedRawSentinelNode<PreciseAllocation>> m_lowerTierFreeList;
-    SentinelLinkedList<IsoCellSet, PackedRawSentinelNode<IsoCellSet>> m_cellSets;
+    SentinelLinkedList<PreciseAllocation, BasicRawSentinelNode<PreciseAllocation>> m_lowerTierFreeList;
+    SentinelLinkedList<IsoCellSet, BasicRawSentinelNode<IsoCellSet>> m_cellSets;
 };
 
 

Source/JavaScriptCore/heap/PreciseAllocation.h

@@ -38,7 +38,7 @@ class SlotVisitor;
 // objects directly using malloc, and put the PreciseAllocation header just before them. We can detect
 // when a HeapCell* is a PreciseAllocation because it will have the MarkedBlock::atomSize / 2 bit set.
 
-class PreciseAllocation : public PackedRawSentinelNode<PreciseAllocation> {
+class PreciseAllocation : public BasicRawSentinelNode<PreciseAllocation> {
 public:
     friend class LLIntOffsetsExtractor;
     friend class IsoSubspace;

Source/JavaScriptCore/heap/Subspace.h

@@ -111,7 +111,7 @@ class Subspace {
 
     BlockDirectory* m_firstDirectory { nullptr };
     BlockDirectory* m_directoryForEmptyAllocation { nullptr }; // Uses the MarkedSpace linked list of blocks.
-    SentinelLinkedList<PreciseAllocation, PackedRawSentinelNode<PreciseAllocation>> m_preciseAllocations;
+    SentinelLinkedList<PreciseAllocation, BasicRawSentinelNode<PreciseAllocation>> m_preciseAllocations;
 
     bool m_isIsoSubspace { false };
     uint8_t m_remainingLowerTierCellCount { 0 };

Source/JavaScriptCore/jit/BaselineJITCode.h

@@ -67,7 +67,7 @@ class JITConstantPool {
         FunctionExpr,
     };
 
-    using Value = CompactPointerTuple<void*, Type>;
+    using Value = JITConstant<Type>;
 
     JITConstantPool() = default;
     JITConstantPool(JITConstantPool&&) = default;

Source/JavaScriptCore/jit/ExecutableMemoryHandle.h

@@ -46,7 +46,7 @@ namespace JSC {
 #if ENABLE(LIBPAS_JIT_HEAP) && ENABLE(JIT)
 DECLARE_ALLOCATOR_WITH_HEAP_IDENTIFIER(ExecutableMemoryHandle);
 class ExecutableMemoryHandle : public ThreadSafeRefCounted<ExecutableMemoryHandle> {
-    WTF_MAKE_FAST_ALLOCATED_WITH_HEAP_IDENTIFIER(ExecutableMemoryHandle);
+    WTF_MAKE_FAST_COMPACT_ALLOCATED_WITH_HEAP_IDENTIFIER(ExecutableMemoryHandle);
 
 public:
     using MemoryPtr = CodePtr<WTF::HandleMemoryPtrTag>;

Source/JavaScriptCore/jit/JITCode.h

@@ -72,6 +72,73 @@ enum class JITType : uint8_t {
 static constexpr unsigned widthOfJITType = 3;
 static_assert(WTF::getMSBSetConstexpr(static_cast<std::underlying_type_t<JITType>>(JITType::FTLJIT)) + 1 == widthOfJITType);
 
+#if USE(JSVALUE64)
+template<typename ByteSizedEnumType>
+class JITConstant {
+    static_assert(sizeof(ByteSizedEnumType) == 1);
+    static constexpr uint64_t typeShift = 48;
+    static constexpr uint64_t typeMask = 0xffull << typeShift;
+    uint64_t m_encodedPointer;
+
+    inline uint64_t encode(void* pointer, ByteSizedEnumType type)
+    {
+        uint64_t pointerBits = bitwise_cast<uint64_t>(pointer);
+        return pointerBits | static_cast<uint64_t>(type) << 48;
+    }
+public:
+    inline JITConstant()
+        : m_encodedPointer(0)
+    { }
+
+    inline JITConstant(void* pointer, ByteSizedEnumType type)
+        : m_encodedPointer(encode(pointer, type))
+    { }
+
+    template<typename OtherPointerType>
+    JITConstant(CompactPointerTuple<OtherPointerType, ByteSizedEnumType> other)
+        : m_encodedPointer(encode(other.pointer(), other.type()))
+    { }
+
+    inline uint32_t hash() const { return computeHash(m_encodedPointer); }
+    inline void* pointer() const { return bitwise_cast<void*>(m_encodedPointer & ~typeMask); }
+    void setPointer(void* pointer) { m_encodedPointer = encode(pointer, type()); }
+    inline ByteSizedEnumType type() const { return static_cast<ByteSizedEnumType>((m_encodedPointer & typeMask) >> typeShift); }
+    void setType(ByteSizedEnumType type) { m_encodedPointer = encode(pointer(), type); }
+
+    friend bool operator==(const JITConstant&, const JITConstant&) = default;
+};
+#else
+template<typename ByteSizedEnumType>
+class JITConstant {
+    void* m_pointer;
+    ByteSizedEnumType m_type;
+public:
+    inline JITConstant()
+        : m_pointer(nullptr)
+    { }
+
+    inline JITConstant(void* pointer, ByteSizedEnumType type)
+        : m_pointer(pointer)
+        , m_type(type)
+    { }
+
+    template<typename OtherPointerType>
+    JITConstant(CompactPointerTuple<OtherPointerType, ByteSizedEnumType> other)
+        : m_pointer(other.pointer())
+        , m_type(other.type())
+    { }
+
+    inline uint32_t hash() const { return computeHash(m_pointer) * 31 + computeHash((unsigned)m_type); }
+    inline void* pointer() const { return m_pointer; }
+    void setPointer(void* pointer) { m_pointer = pointer; }
+    inline ByteSizedEnumType type() const { return m_type; }
+    void setType(ByteSizedEnumType type) { m_type = type; }
+
+    friend bool operator==(const JITConstant&, const JITConstant&) = default;
+};
+#endif
+
+
 class JITCode : public ThreadSafeRefCounted<JITCode> {
 public:
     template<PtrTag tag> using CodeRef = MacroAssemblerCodeRef<tag>;

Source/JavaScriptCore/jit/PolymorphicCallStubRoutine.h

@@ -50,11 +50,11 @@ class PolymorphicCallNode final : public CallLinkInfoBase {
 
     void unlinkImpl(VM&);
 
-    bool hasCallLinkInfo(CallLinkInfo* info) { return m_callLinkInfo.get() == info; }
+    bool hasCallLinkInfo(CallLinkInfo* info) { return m_callLinkInfo == info; }
     void clearCallLinkInfo();
 
 private:
-    PackedPtr<CallLinkInfo> m_callLinkInfo;
+    CallLinkInfo* m_callLinkInfo;
 };
 
 class PolymorphicCallCase {

Source/JavaScriptCore/runtime/ClassInfo.h

@@ -178,6 +178,8 @@ struct MethodTable {
     ClassName::isResizableOrGrowableSharedTypedArray, \
 
 struct CLASS_INFO_ALIGNMENT ClassInfo {
+    WTF_ALLOW_STRUCT_COMPACT_POINTERS;
+
     using CheckJSCastSnippetFunctionPtr = Ref<Snippet> (*)(void);
 
     // A string denoting the class name. Example: "Window".

Source/JavaScriptCore/runtime/JSArray.h

@@ -37,6 +37,7 @@ class JSArray : public JSNonFinalObject {
     friend class LLIntOffsetsExtractor;
     friend class Walker;
     friend class JIT;
+    WTF_ALLOW_COMPACT_POINTERS;
 
 public:
     typedef JSNonFinalObject Base;

Source/JavaScriptCore/runtime/JSCell.h

@@ -90,6 +90,7 @@ template<typename T> void* tryAllocateCell(VM&, GCDeferralContext*, size_t = siz
 #endif
 
 class JSCell : public HeapCell {
+    WTF_ALLOW_COMPACT_POINTERS;
     friend class JSValue;
     friend class MarkedBlock;
     template<typename T>

Source/JavaScriptCore/wasm/WasmTypeDefinition.h

@@ -669,7 +669,7 @@ enum class RTTKind : uint8_t {
 };
 
 class RTT_ALIGNMENT RTT : public ThreadSafeRefCounted<RTT> {
-    WTF_MAKE_FAST_ALLOCATED;
+    WTF_MAKE_FAST_COMPACT_ALLOCATED;
 
 public:
     RTT() = delete;

Source/WTF/wtf/Bag.h

@@ -32,10 +32,8 @@
 
 namespace WTF {
 
-DECLARE_ALLOCATOR_WITH_HEAP_IDENTIFIER(BagNode);
 template<typename T, typename PassedPtrTraits = RawPtrTraits<T>>
 class BagNode {
-    WTF_MAKE_FAST_ALLOCATED_WITH_HEAP_IDENTIFIER(BagNode);
 public:
     using PtrTraits = typename PassedPtrTraits::template RebindTraits<BagNode>;
 
@@ -48,7 +46,7 @@ class BagNode {
     typename PtrTraits::StorageType m_next { nullptr };
 };
 
-template<typename T, typename PassedPtrTraits = RawPtrTraits<T>>
+template<typename T, typename PassedPtrTraits = RawPtrTraits<T>, typename Malloc = FastMalloc>
 class Bag final {
     WTF_MAKE_NONCOPYABLE(Bag);
     WTF_MAKE_FAST_ALLOCATED;
@@ -91,15 +89,17 @@ class Bag final {
         while (head) {
             Node* current = head;
             head = Node::PtrTraits::unwrap(current->m_next);
-            delete current;
+            current->~Node();
+            Malloc::free(current);
         }
         m_head = nullptr;
     }
 
     template<typename... Args>
     T* add(Args&&... args)
     {
-        Node* newNode = new Node(std::forward<Args>(args)...);
+        Node* newNode = static_cast<Node*>(Malloc::malloc(sizeof(Node)));
+        new (NotNull, newNode) Node(std::forward<Args>(args)...);
         newNode->m_next = unwrappedHead();
         m_head = newNode;
         return &newNode->m_item;
@@ -127,7 +127,7 @@ class Bag final {
         friend bool operator==(iterator, iterator) = default;
 
     private:
-        template<typename, typename> friend class WTF::Bag;
+        template<typename, typename, typename> friend class WTF::Bag;
         Node* m_node;
     };