diff --git a/src/google/protobuf/serial_arena.h b/src/google/protobuf/serial_arena.h index 743965f7878e..311c1dd3bb85 100644 --- a/src/google/protobuf/serial_arena.h +++ b/src/google/protobuf/serial_arena.h @@ -88,6 +88,9 @@ struct FirstSerialArena { // used. class PROTOBUF_EXPORT SerialArena { public: + static constexpr size_t kBlockHeaderSize = + ArenaAlignDefault::Ceil(sizeof(ArenaBlock)); + void CleanupList(); size_t FreeStringBlocks() { // On the active block delete all strings skipping the unused instances. @@ -273,6 +276,24 @@ class PROTOBUF_EXPORT SerialArena { std::vector PeekCleanupListForTesting(); private: + friend class ThreadSafeArena; + + // See comments for cached_blocks_. + struct CachedBlock { + // Simple linked list. + CachedBlock* next; + }; + + static constexpr ptrdiff_t kPrefetchForwardsDegree = ABSL_CACHELINE_SIZE * 16; + static constexpr ptrdiff_t kPrefetchBackwardsDegree = ABSL_CACHELINE_SIZE * 6; + + // Constructor is private as only New() should be used. + inline SerialArena(ArenaBlock* b, ThreadSafeArena& parent); + + // Constructors to handle the first SerialArena. + inline explicit SerialArena(ThreadSafeArena& parent); + inline SerialArena(FirstSerialArena, ArenaBlock* b, ThreadSafeArena& parent); + bool MaybeAllocateString(void*& p); ABSL_ATTRIBUTE_RETURNS_NONNULL void* AllocateFromStringBlockFallback(); @@ -288,9 +309,6 @@ class PROTOBUF_EXPORT SerialArena { cleanup::CreateNode(tag, limit_, elem, destructor); } - static constexpr ptrdiff_t kPrefetchForwardsDegree = ABSL_CACHELINE_SIZE * 16; - static constexpr ptrdiff_t kPrefetchBackwardsDegree = ABSL_CACHELINE_SIZE * 6; - // Prefetch the next kPrefetchForwardsDegree bytes after `prefetch_ptr_` and // up to `prefetch_limit_`, if `next` is within kPrefetchForwardsDegree bytes // of `prefetch_ptr_`. @@ -335,9 +353,6 @@ class PROTOBUF_EXPORT SerialArena { } } - private: - friend class ThreadSafeArena; - // Creates a new SerialArena inside mem using the remaining memory as for // future allocations. // The `parent` arena must outlive the serial arena, which is guaranteed @@ -362,6 +377,29 @@ class PROTOBUF_EXPORT SerialArena { std::memory_order_relaxed); } + // Helper getters/setters to handle relaxed operations on atomic variables. + ArenaBlock* head() { return head_.load(std::memory_order_relaxed); } + const ArenaBlock* head() const { + return head_.load(std::memory_order_relaxed); + } + + char* ptr() { return ptr_.load(std::memory_order_relaxed); } + const char* ptr() const { return ptr_.load(std::memory_order_relaxed); } + void set_ptr(char* ptr) { return ptr_.store(ptr, std::memory_order_relaxed); } + PROTOBUF_ALWAYS_INLINE void set_range(char* ptr, char* limit) { + set_ptr(ptr); + prefetch_ptr_ = ptr; + limit_ = limit; + prefetch_limit_ = limit; + } + + void* AllocateAlignedFallback(size_t n); + void* AllocateAlignedWithCleanupFallback(size_t n, size_t align, + void (*destructor)(void*)); + void AddCleanupFallback(void* elem, void (*destructor)(void*)); + inline void AllocateNewBlock(size_t n); + inline void Init(ArenaBlock* b, size_t offset); + // Members are declared here to track sizeof(SerialArena) and hotness // centrally. They are (roughly) laid out in descending order of hotness. @@ -397,46 +435,8 @@ class PROTOBUF_EXPORT SerialArena { // this free list. // `cached_blocks_[i]` points to the free list for blocks of size `8+2^(i+3)`. // The array of freelists is grown when needed in `ReturnArrayMemory()`. - struct CachedBlock { - // Simple linked list. - CachedBlock* next; - }; uint8_t cached_block_length_ = 0; CachedBlock** cached_blocks_ = nullptr; - - // Helper getters/setters to handle relaxed operations on atomic variables. - ArenaBlock* head() { return head_.load(std::memory_order_relaxed); } - const ArenaBlock* head() const { - return head_.load(std::memory_order_relaxed); - } - - char* ptr() { return ptr_.load(std::memory_order_relaxed); } - const char* ptr() const { return ptr_.load(std::memory_order_relaxed); } - void set_ptr(char* ptr) { return ptr_.store(ptr, std::memory_order_relaxed); } - PROTOBUF_ALWAYS_INLINE void set_range(char* ptr, char* limit) { - set_ptr(ptr); - prefetch_ptr_ = ptr; - limit_ = limit; - prefetch_limit_ = limit; - } - - // Constructor is private as only New() should be used. - inline SerialArena(ArenaBlock* b, ThreadSafeArena& parent); - - // Constructors to handle the first SerialArena. - inline explicit SerialArena(ThreadSafeArena& parent); - inline SerialArena(FirstSerialArena, ArenaBlock* b, ThreadSafeArena& parent); - - void* AllocateAlignedFallback(size_t n); - void* AllocateAlignedWithCleanupFallback(size_t n, size_t align, - void (*destructor)(void*)); - void AddCleanupFallback(void* elem, void (*destructor)(void*)); - inline void AllocateNewBlock(size_t n); - inline void Init(ArenaBlock* b, size_t offset); - - public: - static constexpr size_t kBlockHeaderSize = - ArenaAlignDefault::Ceil(sizeof(ArenaBlock)); }; inline PROTOBUF_ALWAYS_INLINE bool SerialArena::MaybeAllocateString(void*& p) {