Skip to content

Commit

Permalink
Merge pull request #58029 from skyoct/feat/server_settings
Browse files Browse the repository at this point in the history 
Add `changeable_without_restart` column to `system.server_settings`
  • Loading branch information
@rschu1ze
rschu1ze committed Jan 8, 2024
2 parents 06274c7 + 8cd8552 commit 6665d23
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 44 deletions.
36 changes: 20 additions & 16 deletions docs/en/operations/system-tables/server_settings.md
View file
Expand Up @@ -14,6 +14,11 @@ Columns:
- `changed` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether a setting was specified in `config.xml`
- `description` ([String](../../sql-reference/data-types/string.md)) — Short server setting description.
- `type` ([String](../../sql-reference/data-types/string.md)) — Server setting value type.
- `changeable_without_restart` ([Enum8](../../sql-reference/data-types/enum.md)) — Whether the setting can be changed at server runtime. Values:
- `'No' `
- `'IncreaseOnly'`
- `'DecreaseOnly'`
- `'Yes'`
- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) - Shows whether a setting is obsolete.

**Example**
Expand All @@ -27,22 +32,21 @@ WHERE name LIKE '%thread_pool%'
```

``` text
┌─name────────────────────────────────────────_─value─_─default─_─changed─_─description──────────────────────────────────────────────────────────────────────────────────────────────────────
───────────────────────────────────_─type───_─is_obsolete─┐
│ max_thread_pool_size │ 10000 │ 10000 │ 1 │ The maximum number of threads that could be allocated from the OS and used for query execution and background operations. │ UInt64 │ 0 │
│ max_thread_pool_free_size │ 1000 │ 1000 │ 0 │ The maximum number of threads that will always stay in a global thread pool once allocated and remain idle in case of insufficient number of tasks. │ UInt64 │ 0 │
│ thread_pool_queue_size │ 10000 │ 10000 │ 0 │ The maximum number of tasks that will be placed in a queue and wait for execution. │ UInt64 │ 0 │
│ max_io_thread_pool_size │ 100 │ 100 │ 0 │ The maximum number of threads that would be used for IO operations │ UInt64 │ 0 │
│ max_io_thread_pool_free_size │ 0 │ 0 │ 0 │ Max free size for IO thread pool. │ UInt64 │ 0 │
│ io_thread_pool_queue_size │ 10000 │ 10000 │ 0 │ Queue size for IO thread pool. │ UInt64 │ 0 │
│ max_active_parts_loading_thread_pool_size │ 64 │ 64 │ 0 │ The number of threads to load active set of data parts (Active ones) at startup. │ UInt64 │ 0 │
│ max_outdated_parts_loading_thread_pool_size │ 32 │ 32 │ 0 │ The number of threads to load inactive set of data parts (Outdated ones) at startup. │ UInt64 │ 0 │
│ max_parts_cleaning_thread_pool_size │ 128 │ 128 │ 0 │ The number of threads for concurrent removal of inactive data parts. │ UInt64 │ 0 │
│ max_backups_io_thread_pool_size │ 1000 │ 1000 │ 0 │ The maximum number of threads that would be used for IO operations for BACKUP queries │ UInt64 │ 0 │
│ max_backups_io_thread_pool_free_size │ 0 │ 0 │ 0 │ Max free size for backups IO thread pool. │ UInt64 │ 0 │
│ backups_io_thread_pool_queue_size │ 0 │ 0 │ 0 │ Queue size for backups IO thread pool. │ UInt64 │ 0 │
└─────────────────────────────────────────────┴───────┴─────────┴─────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────
───────────────────────────────────┴────────┴─────────────┘
┌─name────────────────────────────────────────┬─value─┬─default─┬─changed─┬─description─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─type───┬─changeable_without_restart─┬─is_obsolete─┐
│ max_thread_pool_size │ 10000 │ 10000 │ 0 │ The maximum number of threads that could be allocated from the OS and used for query execution and background operations. │ UInt64 │ No │ 0 │
│ max_thread_pool_free_size │ 1000 │ 1000 │ 0 │ The maximum number of threads that will always stay in a global thread pool once allocated and remain idle in case of insufficient number of tasks. │ UInt64 │ No │ 0 │
│ thread_pool_queue_size │ 10000 │ 10000 │ 0 │ The maximum number of tasks that will be placed in a queue and wait for execution. │ UInt64 │ No │ 0 │
│ max_io_thread_pool_size │ 100 │ 100 │ 0 │ The maximum number of threads that would be used for IO operations │ UInt64 │ No │ 0 │
│ max_io_thread_pool_free_size │ 0 │ 0 │ 0 │ Max free size for IO thread pool. │ UInt64 │ No │ 0 │
│ io_thread_pool_queue_size │ 10000 │ 10000 │ 0 │ Queue size for IO thread pool. │ UInt64 │ No │ 0 │
│ max_active_parts_loading_thread_pool_size │ 64 │ 64 │ 0 │ The number of threads to load active set of data parts (Active ones) at startup. │ UInt64 │ No │ 0 │
│ max_outdated_parts_loading_thread_pool_size │ 32 │ 32 │ 0 │ The number of threads to load inactive set of data parts (Outdated ones) at startup. │ UInt64 │ No │ 0 │
│ max_parts_cleaning_thread_pool_size │ 128 │ 128 │ 0 │ The number of threads for concurrent removal of inactive data parts. │ UInt64 │ No │ 0 │
│ max_backups_io_thread_pool_size │ 1000 │ 1000 │ 0 │ The maximum number of threads that would be used for IO operations for BACKUP queries │ UInt64 │ No │ 0 │
│ max_backups_io_thread_pool_free_size │ 0 │ 0 │ 0 │ Max free size for backups IO thread pool. │ UInt64 │ No │ 0 │
│ backups_io_thread_pool_queue_size │ 0 │ 0 │ 0 │ Queue size for backups IO thread pool. │ UInt64 │ No │ 0 │
└─────────────────────────────────────────────┴───────┴─────────┴─────────┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴────────┴────────────────────────────┴─────────────┘
```

Using of `WHERE changed` can be useful, for example, when you want to check
Expand Down
2 changes: 2 additions & 0 deletions src/Core/ServerSettings.h
View file
Expand Up @@ -113,6 +113,8 @@ namespace DB
M(Bool, validate_tcp_client_information, false, "Validate client_information in the query packet over the native TCP protocol.", 0) \
M(Bool, storage_metadata_write_full_object_key, false, "Write disk metadata files with VERSION_FULL_OBJECT_KEY format", 0) \

/// If you add a setting which can be updated at runtime, please update 'changeable_settings' map in StorageSystemServerSettings.cpp

DECLARE_SETTINGS_TRAITS(ServerSettingsTraits, SERVER_SETTINGS)

struct ServerSettings : public BaseSettings<ServerSettingsTraits>
Expand Down
88 changes: 60 additions & 28 deletions src/Storages/System/StorageSystemServerSettings.cpp
View file
@@ -1,11 +1,15 @@
#include <Storages/System/StorageSystemServerSettings.h>
#include <Core/BackgroundSchedulePool.h>
#include <Core/ServerSettings.h>
#include <DataTypes/DataTypeEnum.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypesNumber.h>
#include <IO/MMappedFileCache.h>
#include <IO/UncompressedCache.h>
#include <Interpreters/Context.h>
#include <Interpreters/ProcessList.h>
#include <Core/ServerSettings.h>
#include <Storages/MarkCache.h>
#include <Storages/MergeTree/MergeTreeBackgroundExecutor.h>
#include <Storages/System/StorageSystemServerSettings.h>


namespace CurrentMetrics
Expand All @@ -18,46 +22,72 @@ namespace CurrentMetrics

namespace DB
{

enum class ChangeableWithoutRestart
{
No,
IncreaseOnly,
DecreaseOnly,
Yes
};

NamesAndTypesList StorageSystemServerSettings::getNamesAndTypes()
{
auto changeable_without_restart_type = std::make_shared<DataTypeEnum8>(
DataTypeEnum8::Values
{
{"No", static_cast<Int8>(ChangeableWithoutRestart::No)},
{"IncreaseOnly", static_cast<Int8>(ChangeableWithoutRestart::IncreaseOnly)},
{"DecreaseOnly", static_cast<Int8>(ChangeableWithoutRestart::DecreaseOnly)},
{"Yes", static_cast<Int8>(ChangeableWithoutRestart::Yes)},
});

return {
{"name", std::make_shared<DataTypeString>()},
{"value", std::make_shared<DataTypeString>()},
{"default", std::make_shared<DataTypeString>()},
{"changed", std::make_shared<DataTypeUInt8>()},
{"description", std::make_shared<DataTypeString>()},
{"type", std::make_shared<DataTypeString>()},
{"is_obsolete", std::make_shared<DataTypeUInt8>()},
{"is_hot_reloadable", std::make_shared<DataTypeUInt8>()}
{"changeable_without_restart", std::move(changeable_without_restart_type)},
{"is_obsolete", std::make_shared<DataTypeUInt8>()}
};
}

void StorageSystemServerSettings::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const
{
// Server settings that have been reloaded from the config file.
std::unordered_map<std::string, std::string> updated = {
{"max_server_memory_usage", std::to_string(total_memory_tracker.getHardLimit())},
{"allow_use_jemalloc_memory", std::to_string(total_memory_tracker.getAllowUseJemallocMmemory())},

{"max_table_size_to_drop", std::to_string(context->getMaxTableSizeToDrop())},
{"max_partition_size_to_drop", std::to_string(context->getMaxPartitionSizeToDrop())},

{"max_concurrent_queries", std::to_string(context->getProcessList().getMaxSize())},
{"max_concurrent_insert_queries", std::to_string(context->getProcessList().getMaxInsertQueriesAmount())},
{"max_concurrent_select_queries", std::to_string(context->getProcessList().getMaxSelectQueriesAmount())},

{"background_buffer_flush_schedule_pool_size", std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundBufferFlushSchedulePoolSize))},
{"background_schedule_pool_size", std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundSchedulePoolSize))},
{"background_message_broker_schedule_pool_size", std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundMessageBrokerSchedulePoolSize))},
{"background_distributed_schedule_pool_size", std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundDistributedSchedulePoolSize))}
/// When the server configuration file is periodically re-loaded from disk, the server components (e.g. memory tracking) are updated
/// with new the setting values but the settings themselves are not stored between re-loads. As a result, if one wants to know the
/// current setting values, one needs to ask the components directly.
std::unordered_map<String, std::pair<String, ChangeableWithoutRestart>> changeable_settings = {
{"max_server_memory_usage", {std::to_string(total_memory_tracker.getHardLimit()), ChangeableWithoutRestart::Yes}},
{"allow_use_jemalloc_memory", {std::to_string(total_memory_tracker.getAllowUseJemallocMmemory()), ChangeableWithoutRestart::Yes}},

{"max_table_size_to_drop", {std::to_string(context->getMaxTableSizeToDrop()), ChangeableWithoutRestart::Yes}},
{"max_partition_size_to_drop", {std::to_string(context->getMaxPartitionSizeToDrop()), ChangeableWithoutRestart::Yes}},

{"max_concurrent_queries", {std::to_string(context->getProcessList().getMaxSize()), ChangeableWithoutRestart::Yes}},
{"max_concurrent_insert_queries", {std::to_string(context->getProcessList().getMaxInsertQueriesAmount()), ChangeableWithoutRestart::Yes}},
{"max_concurrent_select_queries", {std::to_string(context->getProcessList().getMaxSelectQueriesAmount()), ChangeableWithoutRestart::Yes}},

{"background_buffer_flush_schedule_pool_size", {std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundBufferFlushSchedulePoolSize)), ChangeableWithoutRestart::IncreaseOnly}},
{"background_schedule_pool_size", {std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundSchedulePoolSize)), ChangeableWithoutRestart::IncreaseOnly}},
{"background_message_broker_schedule_pool_size", {std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundMessageBrokerSchedulePoolSize)), ChangeableWithoutRestart::IncreaseOnly}},
{"background_distributed_schedule_pool_size", {std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundDistributedSchedulePoolSize)), ChangeableWithoutRestart::IncreaseOnly}},

{"mark_cache_size", {std::to_string(context->getMarkCache()->maxSizeInBytes()), ChangeableWithoutRestart::Yes}},
{"uncompressed_cache_size", {std::to_string(context->getUncompressedCache()->maxSizeInBytes()), ChangeableWithoutRestart::Yes}},
{"index_mark_cache_size", {std::to_string(context->getIndexMarkCache()->maxSizeInBytes()), ChangeableWithoutRestart::Yes}},
{"index_uncompressed_cache_size", {std::to_string(context->getIndexUncompressedCache()->maxSizeInBytes()), ChangeableWithoutRestart::Yes}},
{"mmap_cache_size", {std::to_string(context->getMMappedFileCache()->maxSizeInBytes()), ChangeableWithoutRestart::Yes}}
};

if (context->areBackgroundExecutorsInitialized())
{
updated.insert({"background_pool_size", std::to_string(context->getMergeMutateExecutor()->getMaxThreads())});
updated.insert({"background_move_pool_size", std::to_string(context->getMovesExecutor()->getMaxThreads())});
updated.insert({"background_fetches_pool_size", std::to_string(context->getFetchesExecutor()->getMaxThreads())});
updated.insert({"background_common_pool_size", std::to_string(context->getCommonExecutor()->getMaxThreads())});
changeable_settings.insert({"background_pool_size", {std::to_string(context->getMergeMutateExecutor()->getMaxThreads()), ChangeableWithoutRestart::IncreaseOnly}});
changeable_settings.insert({"background_move_pool_size", {std::to_string(context->getMovesExecutor()->getMaxThreads()), ChangeableWithoutRestart::IncreaseOnly}});
changeable_settings.insert({"background_fetches_pool_size", {std::to_string(context->getFetchesExecutor()->getMaxThreads()), ChangeableWithoutRestart::IncreaseOnly}});
changeable_settings.insert({"background_common_pool_size", {std::to_string(context->getCommonExecutor()->getMaxThreads()), ChangeableWithoutRestart::IncreaseOnly}});
}

const auto & config = context->getConfigRef();
Expand All @@ -67,16 +97,18 @@ void StorageSystemServerSettings::fillData(MutableColumns & res_columns, Context
for (const auto & setting : settings.all())
{
const auto & setting_name = setting.getName();
const auto & it = updated.find(setting_name);

const auto & changeable_settings_it = changeable_settings.find(setting_name);
const bool is_changeable = (changeable_settings_it != changeable_settings.end());

res_columns[0]->insert(setting_name);
res_columns[1]->insert((it != updated.end()) ? it->second : setting.getValueString());
res_columns[1]->insert(is_changeable ? changeable_settings_it->second.first : setting.getValueString());
res_columns[2]->insert(setting.getDefaultValueString());
res_columns[3]->insert(setting.isValueChanged());
res_columns[4]->insert(setting.getDescription());
res_columns[5]->insert(setting.getTypeName());
res_columns[6]->insert(setting.isObsolete());
res_columns[7]->insert((it != updated.end()) ? true : false);
res_columns[6]->insert(is_changeable ? changeable_settings_it->second.second : ChangeableWithoutRestart::No);
res_columns[7]->insert(setting.isObsolete());
}
}

Expand Down

0 comments on commit 6665d23

Please sign in to comment.