Skip to content
This repository has been archived by the owner on Jun 23, 2022. It is now read-only.

feat: tcmalloc memory release improvements #435

Merged
merged 5 commits into from Apr 10, 2020
Merged

feat: tcmalloc memory release improvements #435

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c0f95f0
enhance tcmalloc
hycdong Apr 7, 2020
9567b11
fix perf-counter
hycdong Apr 8, 2020
a3beee1
Merge branch 'master' into tcmalloc_enhancement
hycdong Apr 8, 2020
d99c66b
fix by review
hycdong Apr 9, 2020
3eb30ae
fix comment
hycdong Apr 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
53 changes: 42 additions & 11 deletions src/dist/replication/lib/replica_stub.cpp
View file
Open in desktop
Expand Up @@ -69,16 +69,20 @@ replica_stub::replica_stub(replica_state_subscriber subscriber /*= nullptr*/,
_query_compact_command(nullptr),
_query_app_envs_command(nullptr),
_useless_dir_reserve_seconds_command(nullptr),
_max_reserved_memory_percentage_command(nullptr),
_deny_client(false),
_verbose_client_log(false),
_verbose_commit_log(false),
_gc_disk_error_replica_interval_seconds(3600),
_gc_disk_garbage_replica_interval_seconds(3600),
_release_tcmalloc_memory(false),
_mem_release_max_reserved_mem_percentage(10),
_learn_app_concurrent_count(0),
_fs_manager(false)
{
#ifdef DSN_ENABLE_GPERF
_release_tcmalloc_memory_command = nullptr;
_max_reserved_memory_percentage_command = nullptr;
#endif
_replica_state_subscriber = subscriber;
_is_long_subscriber = is_long_subscriber;
_failure_detector = nullptr;
Expand Down Expand Up @@ -323,6 +327,13 @@ void replica_stub::install_perf_counters()
"recent_write_size_exceed_threshold_count",
COUNTER_TYPE_VOLATILE_NUMBER,
"write size exceed threshold count in the recent period");

#ifdef DSN_ENABLE_GPERF
_counter_tcmalloc_release_memory_size.init_app_counter("eon.replica_stub",
"tcmalloc.release.memory.size",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tcmalloc_release_memory_size

COUNTER_TYPE_NUMBER,
"current tcmalloc release memory size");
#endif
}

void replica_stub::initialize(bool clear /* = false*/)
Expand Down Expand Up @@ -352,6 +363,7 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f
_verbose_commit_log = _options.verbose_commit_log_on_start;
_gc_disk_error_replica_interval_seconds = _options.gc_disk_error_replica_interval_seconds;
_gc_disk_garbage_replica_interval_seconds = _options.gc_disk_garbage_replica_interval_seconds;
_release_tcmalloc_memory = _options.mem_release_enabled;
_mem_release_max_reserved_mem_percentage = _options.mem_release_max_reserved_mem_percentage;

// clear dirs if need
Expand Down Expand Up @@ -673,15 +685,13 @@ void replica_stub::initialize_start()
}

#ifdef DSN_ENABLE_GPERF
if (_options.mem_release_enabled) {
_mem_release_timer_task = tasking::enqueue_timer(
LPC_MEM_RELEASE,
&_tracker,
std::bind(&replica_stub::gc_tcmalloc_memory, this),
std::chrono::milliseconds(_options.mem_release_check_interval_ms),
0,
std::chrono::milliseconds(_options.mem_release_check_interval_ms));
}
_mem_release_timer_task =
tasking::enqueue_timer(LPC_MEM_RELEASE,
&_tracker,
std::bind(&replica_stub::gc_tcmalloc_memory, this),
std::chrono::milliseconds(_options.mem_release_check_interval_ms),
0,
std::chrono::milliseconds(_options.mem_release_check_interval_ms));
#endif

if (_options.duplication_enabled) {
Expand Down Expand Up @@ -2158,6 +2168,15 @@ void replica_stub::open_service()
});

#ifdef DSN_ENABLE_GPERF
_release_tcmalloc_memory_command = ::dsn::command_manager::instance().register_app_command(
{"release-tcmalloc-memory"},
"release-tcmalloc-memory <true|false>",
"release-tcmalloc-memory - control if try to release tcmalloc memory",
[this](const std::vector<std::string> &args) {
return remote_command_set_bool_flag(
_release_tcmalloc_memory, "release-tcmalloc-memory", args);
});

_max_reserved_memory_percentage_command = dsn::command_manager::instance().register_app_command(
{"mem-release-max-reserved-percentage"},
"mem-release-max-reserved-percentage [num | DEFAULT]",
Expand All @@ -2177,7 +2196,7 @@ void replica_stub::open_service()
return result;
}
int32_t percentage = 0;
if (!dsn::buf2int32(args[0], percentage) || percentage <= 0 || percentage >= 100) {
if (!dsn::buf2int32(args[0], percentage) || percentage <= 0 || percentage > 100) {
result = std::string("ERR: invalid arguments");
} else {
_mem_release_max_reserved_mem_percentage = percentage;
Expand Down Expand Up @@ -2311,6 +2330,7 @@ void replica_stub::close()
dsn::command_manager::instance().deregister_command(_query_app_envs_command);
dsn::command_manager::instance().deregister_command(_useless_dir_reserve_seconds_command);
#ifdef DSN_ENABLE_GPERF
dsn::command_manager::instance().deregister_command(_release_tcmalloc_memory_command);
dsn::command_manager::instance().deregister_command(_max_reserved_memory_percentage_command);
#endif

Expand All @@ -2322,7 +2342,10 @@ void replica_stub::close()
_query_compact_command = nullptr;
_query_app_envs_command = nullptr;
_useless_dir_reserve_seconds_command = nullptr;
#ifdef DSN_ENABLE_GPERF
_release_tcmalloc_memory_command = nullptr;
_max_reserved_memory_percentage_command = nullptr;
#endif

if (_config_sync_timer_task != nullptr) {
_config_sync_timer_task->cancel(true);
Expand Down Expand Up @@ -2461,6 +2484,12 @@ static int64_t get_tcmalloc_numeric_property(const char *prop)

void replica_stub::gc_tcmalloc_memory()
{
int64_t tcmalloc_released_bytes = 0;
if (!_release_tcmalloc_memory) {
_counter_tcmalloc_release_memory_size->set(tcmalloc_released_bytes);
return;
}

int64_t total_allocated_bytes =
get_tcmalloc_numeric_property("generic.current_allocated_bytes");
int64_t reserved_bytes = get_tcmalloc_numeric_property("tcmalloc.pageheap_free_bytes");
Expand All @@ -2472,6 +2501,7 @@ void replica_stub::gc_tcmalloc_memory()
total_allocated_bytes * _mem_release_max_reserved_mem_percentage / 100.0;
if (reserved_bytes > max_reserved_bytes) {
int64_t release_bytes = reserved_bytes - max_reserved_bytes;
tcmalloc_released_bytes = release_bytes;
ddebug_f("Memory release started, almost {} bytes will be released", release_bytes);
while (release_bytes > 0) {
// tcmalloc releasing memory will lock page heap, release 1MB at a time to avoid locking
Expand All @@ -2480,6 +2510,7 @@ void replica_stub::gc_tcmalloc_memory()
release_bytes -= 1024 * 1024;
}
}
_counter_tcmalloc_release_memory_size->set(tcmalloc_released_bytes);
}
#endif

Expand Down
7 changes: 7 additions & 0 deletions src/dist/replication/lib/replica_stub.h
View file
Open in desktop
Expand Up @@ -328,13 +328,17 @@ class replica_stub : public serverlet<replica_stub>, public ref_counter
dsn_handle_t _query_compact_command;
dsn_handle_t _query_app_envs_command;
dsn_handle_t _useless_dir_reserve_seconds_command;
#ifdef DSN_ENABLE_GPERF
dsn_handle_t _release_tcmalloc_memory_command;
hycdong marked this conversation as resolved.
Show resolved Hide resolved
dsn_handle_t _max_reserved_memory_percentage_command;
#endif

bool _deny_client;
bool _verbose_client_log;
bool _verbose_commit_log;
int32_t _gc_disk_error_replica_interval_seconds;
int32_t _gc_disk_garbage_replica_interval_seconds;
bool _release_tcmalloc_memory;
int32_t _mem_release_max_reserved_mem_percentage;

// we limit LT_APP max concurrent count, because nfs service implementation is
Expand Down Expand Up @@ -415,6 +419,9 @@ class replica_stub : public serverlet<replica_stub>, public ref_counter

perf_counter_wrapper _counter_recent_write_size_exceed_threshold_count;

#ifdef DSN_ENABLE_GPERF
perf_counter_wrapper _counter_tcmalloc_release_memory_size;
#endif
dsn::task_tracker _tracker;
};
} // namespace replication
Expand Down