From 72b4de59cdecb5bc49644ec1559e5e720ad9d377 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Mon, 3 Jun 2024 17:09:19 +0200 Subject: [PATCH 1/6] add allocation release --- include/api/CJsonOutputWriter.h | 3 +++ include/core/CBoostJsonWriterBase.h | 11 ++++++++--- include/core/CScopedBoostJsonPoolAllocator.h | 9 +++++++-- lib/api/CJsonOutputWriter.cc | 4 ++++ 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/include/api/CJsonOutputWriter.h b/include/api/CJsonOutputWriter.h index d5bae88698..e89e259c7c 100644 --- a/include/api/CJsonOutputWriter.h +++ b/include/api/CJsonOutputWriter.h @@ -273,6 +273,9 @@ class API_EXPORT CJsonOutputWriter { //! \p allocatorName A unique identifier for the allocator void pushAllocator(const std::string& allocatorName); + //! release the allocator + void releaseAllocator(const std::string& allocatorName); + //! revert to using the previous allocator for JSON output processing void popAllocator(); diff --git a/include/core/CBoostJsonWriterBase.h b/include/core/CBoostJsonWriterBase.h index 1cd9a76c0e..49becdbd4b 100644 --- a/include/core/CBoostJsonWriterBase.h +++ b/include/core/CBoostJsonWriterBase.h @@ -24,10 +24,7 @@ #include #include -#include -#include #include -#include #include namespace json = boost::json; @@ -134,6 +131,14 @@ class CBoostJsonWriterBase { return this->getAllocator()->get(); } + void releaseAllocator(const std::string& allocatorName) { + if (m_AllocatorCache.find(allocatorName) != m_AllocatorCache.end()) { + TPoolAllocatorPtr allocator = m_AllocatorCache[allocatorName]; + allocator.reset(); + m_AllocatorCache.erase(allocatorName); + } + } + bool isComplete() const { bool ret = m_Levels.empty() || m_Levels.top() == 0; return ret; diff --git a/include/core/CScopedBoostJsonPoolAllocator.h b/include/core/CScopedBoostJsonPoolAllocator.h index 6b29f0e4f9..78ab678b47 100644 --- a/include/core/CScopedBoostJsonPoolAllocator.h +++ b/include/core/CScopedBoostJsonPoolAllocator.h @@ -12,6 +12,7 @@ #define INCLUDED_ml_core_CScopedBoostJsonPoolAllocator_h #include +#include namespace ml { namespace core { @@ -31,14 +32,18 @@ class CScopedBoostJsonPoolAllocator { //! \p allocatorName Unique identifier for the allocator //! \p jsonOutputWriter JSON output writer that will make use of the allocator CScopedBoostJsonPoolAllocator(const std::string& allocatorName, T& writer) - : m_Writer(writer) { + : m_Writer(writer), m_AllocatorName(allocatorName) { m_Writer.pushAllocator(allocatorName); } - ~CScopedBoostJsonPoolAllocator() { m_Writer.popAllocator(); } + ~CScopedBoostJsonPoolAllocator() { + m_Writer.popAllocator(); + m_Writer.releaseAllocator(m_AllocatorName); + } private: T& m_Writer; + std::string m_AllocatorName; }; } } diff --git a/lib/api/CJsonOutputWriter.cc b/lib/api/CJsonOutputWriter.cc index 2a7a987c23..44e5a5699d 100644 --- a/lib/api/CJsonOutputWriter.cc +++ b/lib/api/CJsonOutputWriter.cc @@ -850,6 +850,10 @@ void CJsonOutputWriter::pushAllocator(const std::string& allocatorName) { m_Writer.pushAllocator(allocatorName); } +void CJsonOutputWriter::releaseAllocator(const std::string& allocatorName) { + m_Writer.releaseAllocator(allocatorName); +} + void CJsonOutputWriter::popAllocator() { m_Writer.popAllocator(); } From a3a2cdda8fe20445d17913499896b1c552b5dabb Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Tue, 4 Jun 2024 09:47:28 +0200 Subject: [PATCH 2/6] formatting --- include/core/CScopedBoostJsonPoolAllocator.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/core/CScopedBoostJsonPoolAllocator.h b/include/core/CScopedBoostJsonPoolAllocator.h index 78ab678b47..5c7dbdd41d 100644 --- a/include/core/CScopedBoostJsonPoolAllocator.h +++ b/include/core/CScopedBoostJsonPoolAllocator.h @@ -36,10 +36,10 @@ class CScopedBoostJsonPoolAllocator { m_Writer.pushAllocator(allocatorName); } - ~CScopedBoostJsonPoolAllocator() { - m_Writer.popAllocator(); + ~CScopedBoostJsonPoolAllocator() { + m_Writer.popAllocator(); m_Writer.releaseAllocator(m_AllocatorName); - } + } private: T& m_Writer; From 0f2e38d99bd7136a64951c9b24e911fe80b26fe5 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 5 Jun 2024 10:10:30 +0200 Subject: [PATCH 3/6] Documentation --- docs/CHANGELOG.asciidoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 4b0ff237d3..fc2b10221c 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -28,6 +28,13 @@ //=== Regressions +== {es} version 8.14.1 + +=== Bug Fixes + +* Improve memory allocation management for JSON processing to reduce memory usage. + (See {ml-pull}2676[#2676].) + == {es} version 8.14.0 === Bug Fixes From e743f78c83f7754c93eb15de10e151788db26b64 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 5 Jun 2024 10:11:11 +0200 Subject: [PATCH 4/6] Documentation --- docs/CHANGELOG.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index fc2b10221c..43b8064fe4 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -30,7 +30,7 @@ == {es} version 8.14.1 -=== Bug Fixes +=== Enhancements * Improve memory allocation management for JSON processing to reduce memory usage. (See {ml-pull}2676[#2676].) From a7979d173673d5e58827af24f0898ea715e8437a Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 6 Jun 2024 09:58:45 +0200 Subject: [PATCH 5/6] rename releaseAllocator method and update changelog --- docs/CHANGELOG.asciidoc | 3 ++- include/api/CJsonOutputWriter.h | 4 ++-- include/core/CBoostJsonWriterBase.h | 2 +- include/core/CScopedBoostJsonPoolAllocator.h | 2 +- lib/api/CJsonOutputWriter.cc | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 43b8064fe4..601c43cd0e 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -33,7 +33,8 @@ === Enhancements * Improve memory allocation management for JSON processing to reduce memory usage. - (See {ml-pull}2676[#2676].) + (See {ml-pull}2679[#2679].) + == {es} version 8.14.0 diff --git a/include/api/CJsonOutputWriter.h b/include/api/CJsonOutputWriter.h index e89e259c7c..39b9f444e7 100644 --- a/include/api/CJsonOutputWriter.h +++ b/include/api/CJsonOutputWriter.h @@ -273,8 +273,8 @@ class API_EXPORT CJsonOutputWriter { //! \p allocatorName A unique identifier for the allocator void pushAllocator(const std::string& allocatorName); - //! release the allocator - void releaseAllocator(const std::string& allocatorName); + //! remove allocator from cache + void removeAllocator(const std::string& allocatorName); //! revert to using the previous allocator for JSON output processing void popAllocator(); diff --git a/include/core/CBoostJsonWriterBase.h b/include/core/CBoostJsonWriterBase.h index 49becdbd4b..e4cd3e8620 100644 --- a/include/core/CBoostJsonWriterBase.h +++ b/include/core/CBoostJsonWriterBase.h @@ -131,7 +131,7 @@ class CBoostJsonWriterBase { return this->getAllocator()->get(); } - void releaseAllocator(const std::string& allocatorName) { + void removeAllocator(const std::string& allocatorName) { if (m_AllocatorCache.find(allocatorName) != m_AllocatorCache.end()) { TPoolAllocatorPtr allocator = m_AllocatorCache[allocatorName]; allocator.reset(); diff --git a/include/core/CScopedBoostJsonPoolAllocator.h b/include/core/CScopedBoostJsonPoolAllocator.h index 5c7dbdd41d..1febdff495 100644 --- a/include/core/CScopedBoostJsonPoolAllocator.h +++ b/include/core/CScopedBoostJsonPoolAllocator.h @@ -38,7 +38,7 @@ class CScopedBoostJsonPoolAllocator { ~CScopedBoostJsonPoolAllocator() { m_Writer.popAllocator(); - m_Writer.releaseAllocator(m_AllocatorName); + m_Writer.removeAllocator(m_AllocatorName); } private: diff --git a/lib/api/CJsonOutputWriter.cc b/lib/api/CJsonOutputWriter.cc index 44e5a5699d..63f8f2bf50 100644 --- a/lib/api/CJsonOutputWriter.cc +++ b/lib/api/CJsonOutputWriter.cc @@ -850,8 +850,8 @@ void CJsonOutputWriter::pushAllocator(const std::string& allocatorName) { m_Writer.pushAllocator(allocatorName); } -void CJsonOutputWriter::releaseAllocator(const std::string& allocatorName) { - m_Writer.releaseAllocator(allocatorName); +void CJsonOutputWriter::removeAllocator(const std::string& allocatorName) { + m_Writer.removeAllocator(allocatorName); } void CJsonOutputWriter::popAllocator() { From 0085a3e4e973dfad2d4f2ad3f62ce8d11d6d0732 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Mon, 10 Jun 2024 10:07:34 +0200 Subject: [PATCH 6/6] make map lookup more efficient --- include/core/CBoostJsonWriterBase.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/core/CBoostJsonWriterBase.h b/include/core/CBoostJsonWriterBase.h index e4cd3e8620..e0bee383c6 100644 --- a/include/core/CBoostJsonWriterBase.h +++ b/include/core/CBoostJsonWriterBase.h @@ -132,10 +132,10 @@ class CBoostJsonWriterBase { } void removeAllocator(const std::string& allocatorName) { - if (m_AllocatorCache.find(allocatorName) != m_AllocatorCache.end()) { - TPoolAllocatorPtr allocator = m_AllocatorCache[allocatorName]; - allocator.reset(); - m_AllocatorCache.erase(allocatorName); + auto allocator = m_AllocatorCache.find(allocatorName); + if (allocator != m_AllocatorCache.end()) { + allocator->second.reset(); + m_AllocatorCache.erase(allocator); } }