From b175b02c3e298324d179c35384bbc7e7b20f1af8 Mon Sep 17 00:00:00 2001 From: AlexPeshkoff Date: Fri, 5 Oct 2018 17:24:42 +0300 Subject: [PATCH] Backported CORE-5918: Memory pool statistics is not accurate --- src/common/classes/alloc.cpp | 262 +++++++++++++++++++++++++---------- 1 file changed, 186 insertions(+), 76 deletions(-) diff --git a/src/common/classes/alloc.cpp b/src/common/classes/alloc.cpp index 1a02e0fd6d3..4c39241f88c 100644 --- a/src/common/classes/alloc.cpp +++ b/src/common/classes/alloc.cpp @@ -54,6 +54,7 @@ #include "../common/classes/vector.h" #include "../common/classes/RefMutex.h" #include "../common/os/os_utils.h" +#include "../common/os/fbsyslog.h" #include "gen/iberror.h" #ifdef USE_VALGRIND @@ -123,6 +124,8 @@ size_t delayedExtentsPos = 0; // Could slowdown pool significantly ! //#define VALIDATE_POOL +typedef Firebird::AtomicCounter::counter_type StatInt; + // We cache this amount of extents to avoid memory mapping overhead const int MAP_CACHE_SIZE = 16; // == 1 MB const size_t DEFAULT_ALLOCATION = 65536; @@ -148,6 +151,36 @@ void corrupt(const char* text) throw () #endif } +Firebird::Mutex* cache_mutex = NULL; +int dev_zero_fd = 0; + +#if defined(WIN_NT) +size_t get_page_size() +{ + SYSTEM_INFO info; + GetSystemInfo(&info); + return info.dwPageSize; +} +#else +size_t get_page_size() +{ + return sysconf(_SC_PAGESIZE); +} +#endif + + +inline size_t get_map_page_size() +{ + static volatile size_t map_page_size = 0; + if (!map_page_size) + { + Firebird::MutexLockGuard guard(*cache_mutex, "get_map_page_size"); + if (!map_page_size) + map_page_size = get_page_size(); + } + return map_page_size; +} + } // anonymous namespace namespace Firebird { @@ -223,6 +256,7 @@ class MemHeader static const size_t MEM_MASK = 0x7; static const size_t MEM_HUGE = 0x1; static const size_t MEM_REDIRECT = 0x2; + static const size_t MEM_EXTENT = 0x4; static const unsigned OFFSET_SHIFT = 16; enum HugeBlock {HUGE_BLOCK}; @@ -302,6 +336,23 @@ class MemHeader return hdrLength & MEM_REDIRECT; } + void setExtent() + { + fb_assert(!isExtent()); + hdrLength |= MEM_EXTENT; + } + + void resetExtent() + { + fb_assert(isExtent()); + hdrLength &= ~MEM_EXTENT; + } + + bool isExtent() const + { + return hdrLength & MEM_EXTENT; + } + void assertBig() { fb_assert(hdrLength & MEM_HUGE); @@ -320,9 +371,10 @@ class MemHeader if (!filter) { - if (used) + if (used || redirected()) { - fprintf(file, "USED %p: size=%" SIZEFORMAT " allocated at %s:%d", + fprintf(file, "%s %p: size=%" SIZEFORMAT " allocated at %s:%d", + isExtent() ? "EXTN" : redirected() ? "RDIR" : "USED", this, getSize(), fileName, lineNumber); } else @@ -336,6 +388,12 @@ class MemHeader } } #endif + + void validate(MemPool* p, StatInt& vUse) throw () + { + if (p == pool && !isExtent()) + vUse += getSize(); + } }; class MemBlock : public MemHeader @@ -385,8 +443,28 @@ class MemBaseHunk spaceRemaining -= size; } -#ifdef MEM_DEBUG public: + void validate(MemPool* pool, size_t hdr, StatInt& vMap, StatInt& vUse) throw () + { + if (length >= DEFAULT_ALLOCATION) + { + fb_assert(length == DEFAULT_ALLOCATION); + vMap += length; + } + + UCHAR* m = ((UCHAR*) this) + hdr; + while (m < memory) + { + MemBlock* block = (MemBlock*)m; + block->validate(pool, vUse); + m += block->getSize(); + } + + if (next) + next->validate(pool, hdr, vMap, vUse); + } + +#ifdef MEM_DEBUG void print_memory(UCHAR* m, FILE* file, MemPool* pool, bool used_only, const char* filter_path, const size_t filter_len) { @@ -443,7 +521,6 @@ class MemSmallHunk : public MemBaseHunk } #endif -private: static size_t hdrSize() { return MEM_ALIGN(sizeof(MemSmallHunk)); @@ -516,7 +593,6 @@ class MemMediumHunk : public MemBaseHunk } #endif -private: static size_t hdrSize() { return MEM_ALIGN(sizeof(MemMediumHunk)); @@ -555,11 +631,14 @@ class MemBigHunk return MEM_ALIGN(sizeof(MemBigHunk)); } - void validate() + void validate(MemPool* pool, StatInt& vMap, StatInt& vUse) { SemiDoubleLink::validate(this); block->assertBig(); fb_assert(block->getSize() + hdrSize() == length); + + vMap += FB_ALIGN(length, get_map_page_size()); + block->validate(pool, vUse); } }; @@ -1485,7 +1564,7 @@ class LinkedList void decrUsage(MemSmallHunk*, MemPool*) { } - static void validate(MemBlock* block, unsigned length) throw () + static void validate(MemBlock* block, unsigned length) { for (; block; block = block->next) { @@ -1521,7 +1600,7 @@ class DoubleLinkedList void putElement(MemBlock** to, MemBlock* block); - static void validate(MemBlock* block, unsigned length) throw () + static void validate(MemBlock* block, unsigned length) { for (; block; block = block->next) { @@ -1612,10 +1691,13 @@ class FreeObjects } #endif - void validate(void) throw () + void validate(MemPool* pool, StatInt& vMap, StatInt& vUse) { for (unsigned int slot = 0; slot < Limits::TOTAL_ELEMENTS; ++slot) ListBuilder::validate(freeObjects[slot], Limits::getSize(slot)); + + if (currentExtent) + currentExtent->validate(pool, currentExtent->hdrSize(), vMap, vUse); } private: @@ -1671,15 +1753,31 @@ class MemPool Validator(MemPool* p) : m_pool(p) { - m_pool->validate(); + validate(); } ~Validator() { - m_pool->validate(); + validate(); } + private: MemPool* m_pool; + + void validate() + { + if (m_pool) + { + char buf[256]; + if (!m_pool->validate(buf, sizeof(buf))) + { + Syslog::Record(Syslog::Warning, buf); +#ifdef MEM_DEBUG + m_pool->print_contents("validate.failed", 0, NULL); +#endif + } + } + } }; #else class Validator @@ -1690,7 +1788,7 @@ class MemPool #endif // VALIDATE_POOL MemBlock* alloc(size_t from, size_t& length, bool flagRedirect) throw (OOM_EXCEPTION); - void releaseBlock(MemBlock *block) throw (); + void releaseBlock(MemBlock *block, bool flagDecr) throw (); public: void* allocate(size_t size ALLOC_PARAMS) throw (OOM_EXCEPTION); @@ -1699,7 +1797,7 @@ class MemPool private: virtual void memoryIsExhausted(void) throw (OOM_EXCEPTION); void* allocRaw(size_t length) throw (OOM_EXCEPTION); - static void release(void* block, bool flagDecr) throw (); + static void releaseMemory(void* block, bool flagExtent) throw (); static void releaseRaw(bool destroying, void *block, size_t size, bool use_cache = true) throw (); void* getExtent(size_t from, size_t& to) throw (OOM_EXCEPTION); @@ -1723,7 +1821,7 @@ class MemPool static void globalFree(void* block) throw (); static void deallocate(void* block) throw (); - void validate(void) throw (); + bool validate(char* buf, FB_SIZE_T size); // Create memory pool instance static MemPool* createPool(MemPool* parent, MemoryStats& stats); @@ -1855,7 +1953,7 @@ MemBlock* FreeObjects::newBlock(MemPool* pool, unsigned slo currentExtent->spaceRemaining > ListBuilder::MEM_OVERHEAD) { unsigned sl1 = Limits::getSlot(currentExtent->spaceRemaining, SLOT_FREE); - if (sl1 == ~0) + if (sl1 == ~0u) break; unsigned size1 = Limits::getSize(sl1); @@ -1895,42 +1993,9 @@ GlobalPtr forceCreationOfDefaultMemoryPool; MemoryPool* MemoryPool::defaultMemoryManager = NULL; MemoryStats* MemoryPool::default_stats_group = NULL; -Mutex* cache_mutex = NULL; MemPool* MemPool::defaultMemPool = NULL; -namespace { - -volatile size_t map_page_size = 0; -int dev_zero_fd = 0; - -#if defined(WIN_NT) -size_t get_page_size() -{ - SYSTEM_INFO info; - GetSystemInfo(&info); - return info.dwPageSize; -} -#else -size_t get_page_size() -{ - return sysconf(_SC_PAGESIZE); -} -#endif - -inline size_t get_map_page_size() -{ - if (!map_page_size) - { - MutexLockGuard guard(*cache_mutex, "get_map_page_size"); - if (!map_page_size) - map_page_size = get_page_size(); - } - return map_page_size; -} - -} - // Initialize process memory pool (called from InstanceControl). void MemoryPool::init() @@ -2052,12 +2117,18 @@ MemPool::~MemPool(void) releaseRaw(pool_destroying, hunk, hunk->length); } - // release blocks redirected to parent - while (parentRedirected.getCount()) + if (parent) { - MemBlock* block = parentRedirected.pop(); - block->resetRedirect(parent); - parent->releaseBlock(block); + // release blocks redirected to parent +#ifdef VALIDATE_POOL + MutexLockGuard guard(parent->mutex, FB_FUNCTION); +#endif + while (parentRedirected.getCount()) + { + MemBlock* block = parentRedirected.pop(); + block->resetRedirect(parent); + parent->releaseBlock(block, false); + } } #ifdef MEM_DEBUG @@ -2150,8 +2221,6 @@ MemBlock* MemPool::alloc(size_t from, size_t& length, bool flagRedirect) throw ( MutexEnsureUnlock guard(mutex, "MemPool::alloc"); guard.enter(); - Validator vld(this); - // If this is a small block, look for it there MemBlock* block = smallObjects.allocateBlock(this, from, length); @@ -2179,7 +2248,7 @@ MemBlock* MemPool::alloc(size_t from, size_t& length, bool flagRedirect) throw ( else // worst case - very low possibility { guard.leave(); - parent->releaseBlock(block); + parent->releaseBlock(block, false); guard.enter(); } } @@ -2238,6 +2307,11 @@ MemBlock* MemPool::allocate2(size_t from, size_t& size void* MemPool::allocate(size_t size ALLOC_PARAMS) throw (OOM_EXCEPTION) { +#ifdef VALIDATE_POOL + MutexLockGuard guard(mutex, "MemPool::allocate"); + Validator vld(this); +#endif + MemBlock* memory = allocate2(0, size ALLOC_PASS_ARGS); increment_usage(memory->getSize()); @@ -2246,13 +2320,19 @@ void* MemPool::allocate(size_t size ALLOC_PARAMS) throw (OOM_EXCEPTION) } -void MemPool::release(void* object, bool flagDecr) throw () +void MemPool::releaseMemory(void* object, bool flagExtent) throw () { if (object) { MemBlock* block = (MemBlock*) ((UCHAR*) object - offsetof(MemBlock, body)); MemPool* pool = block->pool; +#ifdef VALIDATE_POOL + MutexLockGuard guard(pool->mutex, "MemPool::releaseMemory"); +#endif + if (flagExtent) + block->resetExtent(); + #ifdef USE_VALGRIND // Synchronize delayed free queue using pool mutex MutexLockGuard guard(pool->mutex, "MemPool::deallocate USE_VALGRIND"); @@ -2297,18 +2377,14 @@ void MemPool::release(void* object, bool flagDecr) throw () pool->delayedFreePos = 0; #endif - size_t size = block->getSize(); #ifdef DEBUG_GDS_ALLOC block->fileName = NULL; #endif - pool->releaseBlock(block); - - if (flagDecr) - pool->decrement_usage(size); + pool->releaseBlock(block, !flagExtent); } } -void MemPool::releaseBlock(MemBlock* block) throw () +void MemPool::releaseBlock(MemBlock* block, bool decrUsage) throw () { if (block->pool != this) corrupt("bad block released"); @@ -2324,10 +2400,13 @@ void MemPool::releaseBlock(MemBlock* block) throw () --blocksActive; const size_t length = block->getSize(); - MutexEnsureUnlock guard(mutex, "MemPool::release"); + MutexEnsureUnlock guard(mutex, "MemPool::releaseBlock"); guard.enter(); - Validator vld(this); + Validator vld(decrUsage ? this : NULL); + + if (decrUsage) + decrement_usage(length); // If length is less than threshold, this is a small block if (smallObjects.deallocateBlock(block)) @@ -2341,8 +2420,11 @@ void MemPool::releaseBlock(MemBlock* block) throw () parentRedirected.remove(pos); guard.leave(); +#ifdef VALIDATE_POOL + MutexLockGuard guard(parent->mutex, "MemPool::releaseBlock /parent"); +#endif block->resetRedirect(parent); - parent->releaseBlock(block); + parent->releaseBlock(block, false); return; } @@ -2359,7 +2441,7 @@ void MemPool::releaseBlock(MemBlock* block) throw () MemBigHunk* hunk = (MemBigHunk*)(((UCHAR*)block) - MemBigHunk::hdrSize()); SemiDoubleLink::remove(hunk); - decrement_mapping(hunk->length); + decrement_mapping(FB_ALIGN(hunk->length, get_map_page_size())); releaseRaw(pool_destroying, hunk, hunk->length, false); } @@ -2450,7 +2532,11 @@ void* MemPool::allocRaw(size_t size) throw (OOM_EXCEPTION) void* MemPool::getExtent(size_t from, size_t& to) throw(OOM_EXCEPTION) // pass desired minimum size, return actual extent size { +#ifdef VALIDATE_POOL + MutexLockGuard guard(mutex, "MemPool::getExtent"); +#endif MemBlock* extent = allocate2(from, to ALLOC_ARGS); + extent->setExtent(); return &extent->body; } @@ -2458,7 +2544,7 @@ void* MemPool::getExtent(size_t from, size_t& to) throw(OOM_EXCEPTION) // pass void MemPool::releaseExtent(bool destroying, void* block, size_t size, MemPool* pool) throw () { if (size < DEFAULT_ALLOCATION) - release(block, false); + releaseMemory(block, true); else { if (pool) @@ -2576,7 +2662,7 @@ void* MemoryPool::calloc(size_t size ALLOC_PARAMS) throw (OOM_EXCEPTION) void MemPool::deallocate(void* block) throw () { - release(block, true); + releaseMemory(block, false); } void MemPool::deletePool(MemPool* pool) @@ -2584,16 +2670,35 @@ void MemPool::deletePool(MemPool* pool) delete pool; } -void MemPool::validate(void) throw () +bool MemPool::validate(char* buf, FB_SIZE_T size) { - smallObjects.validate(); - mediumObjects.validate(); + StatInt vMap = 0, vUse = 0; + + smallObjects.validate(this, vMap, vUse); + mediumObjects.validate(this, vMap, vUse); // validate big objects for (MemBigHunk* h = bigHunks; h; h = h->next) + h->validate(this, vMap, vUse); + + // validate blocks redirected to parent + for (FB_SIZE_T n = 0; n < parentRedirected.getCount(); ++n) { - h->validate(); + MemBlock* b = parentRedirected[n]; + if (!b->isExtent()) + vUse += parentRedirected[n]->getSize(); } + + if (vMap != mapped_memory.value() || vUse != used_memory.value()) + { + char buf[256]; + fb_utils::snprintf(buf, sizeof(buf), "Memory statistics does not match pool: " + "mapped=%" SQUADFORMAT "(%" SQUADFORMAT " st), used=%" SQUADFORMAT "(%" SQUADFORMAT " st)", + SINT64(vMap), SINT64(mapped_memory.value()), SINT64(vUse), SINT64(used_memory.value())); + return false; + } + + return true; } #ifdef MEM_DEBUG @@ -2607,7 +2712,6 @@ void MemPool::print_contents(const char* filename, unsigned flags, const char* f fclose(out); } - // This member function can't be const because there are calls to the mutex. void MemPool::print_contents(FILE* file, unsigned flags, const char* filter_path) throw () { @@ -2615,8 +2719,14 @@ void MemPool::print_contents(FILE* file, unsigned flags, const char* filter_path MutexLockGuard guard(mutex, "MemPool::print_contents"); - fprintf(file, "********* Printing contents of pool %p (parent %p) used=%ld mapped=%ld\n", - this, parent, (long) used_memory.value(), (long) mapped_memory.value()); + fprintf(file, "********* Printing contents of pool %p (parent %p) used=%" SQUADFORMAT " mapped=%" SQUADFORMAT "\n", + this, parent, SINT64(used_memory.value()), SINT64(mapped_memory.value())); + + char buf[256]; + if (!validate(buf, sizeof(buf))) + { + fprintf(file, "%s\n", buf); + } if (!used_only) {