Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

last_successful_update_time in system.dictionaries #9394

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 6 additions & 1 deletion dbms/src/Interpreters/ExternalLoader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -696,6 +696,7 @@ class ExternalLoader::LoadingDispatcher : private boost::noncopyable
result.object = object;
result.exception = exception;
result.loading_start_time = loading_start_time;
result.last_successful_update_time = last_successful_update_time;
result.loading_duration = loadingDuration();
result.origin = object_config.path;
result.repository_name = object_config.repository_name;
Expand All @@ -713,6 +714,7 @@ class ExternalLoader::LoadingDispatcher : private boost::noncopyable
ObjectConfig object_config;
TimePoint loading_start_time;
TimePoint loading_end_time;
TimePoint last_successful_update_time;
size_t state_id = 0; /// Index of the current state of this `info`, this index is incremented every loading.
size_t loading_id = 0; /// The value which will be stored in `state_id` after finishing the current loading.
size_t error_count = 0; /// Numbers of errors since last successful loading.
Expand Down Expand Up @@ -1010,7 +1012,10 @@ class ExternalLoader::LoadingDispatcher : private boost::noncopyable

info->exception = new_exception;
info->error_count = error_count;
info->loading_end_time = std::chrono::system_clock::now();
const auto current_time = std::chrono::system_clock::now();
info->loading_end_time = current_time;
if (!info->exception)
info->last_successful_update_time = current_time;
info->state_id = info->loading_id;
info->next_update_time = next_update_time;
}
Expand Down
1 change: 1 addition & 0 deletions dbms/src/Interpreters/ExternalLoader.h
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ class ExternalLoader
LoadablePtr object;
String origin;
TimePoint loading_start_time;
TimePoint last_successful_update_time;
Duration loading_duration;
std::exception_ptr exception;
std::string repository_name;
Expand Down
2 changes: 2 additions & 0 deletions dbms/src/Storages/System/StorageSystemDictionaries.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ NamesAndTypesList StorageSystemDictionaries::getNamesAndTypes()
{"lifetime_min", std::make_shared<DataTypeUInt64>()},
{"lifetime_max", std::make_shared<DataTypeUInt64>()},
{"loading_start_time", std::make_shared<DataTypeDateTime>()},
{"last_successful_update_time", std::make_shared<DataTypeDateTime>()},
{"loading_duration", std::make_shared<DataTypeFloat32>()},
//{ "creation_time", std::make_shared<DataTypeDateTime>() },
{"last_exception", std::make_shared<DataTypeString>()}
Expand Down Expand Up @@ -112,6 +113,7 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, const Con
}

res_columns[i++]->insert(static_cast<UInt64>(std::chrono::system_clock::to_time_t(load_result.loading_start_time)));
res_columns[i++]->insert(static_cast<UInt64>(std::chrono::system_clock::to_time_t(load_result.last_successful_update_time)));
res_columns[i++]->insert(std::chrono::duration_cast<std::chrono::duration<float>>(load_result.loading_duration).count());

if (last_exception)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,12 @@ def get_loading_start_time(dictionary_name):
return None
return time.strptime(s, "%Y-%m-%d %H:%M:%S")

def get_last_successful_update_time(dictionary_name):
s = instance.query("SELECT last_successful_update_time FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n")
if s == "0000-00-00 00:00:00":
return None
return time.strptime(s, "%Y-%m-%d %H:%M:%S")


def get_loading_duration(dictionary_name):
return float(instance.query("SELECT loading_duration FROM system.dictionaries WHERE name='" + dictionary_name + "'"))
Expand Down Expand Up @@ -92,6 +98,9 @@ def test_reload_while_loading(started_cluster):

# This time loading should finish quickly.
assert get_status('slow') == "LOADED"

last_successful_update_time = get_last_successful_update_time('slow')
assert last_successful_update_time > start_time
assert query("SELECT dictGetInt32('slow', 'a', toUInt64(5))") == "6\n"


Expand Down