Skip to content

branch-2.1: [chore](binlog) add ingesting/downloading binlog latency metrics #48599 #49002

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 1 commit into from
Mar 14, 2025
Merged

branch-2.1: [chore](binlog) add ingesting/downloading binlog latency metrics #48599 #49002

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions be/src/http/action/download_binlog_action.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include "olap/storage_engine.h"
#include "olap/tablet_manager.h"
#include "runtime/exec_env.h"
#include "util/stopwatch.hpp"

namespace doris {

Expand All @@ -47,6 +48,12 @@ const std::string kSegmentIndexParameter = "segment_index";
const std::string kSegmentIndexIdParameter = "segment_index_id";
const std::string kAcquireMD5Parameter = "acquire_md5";

bvar::LatencyRecorder g_get_binlog_info_latency("doris_download_binlog", "get_binlog_info");
bvar::LatencyRecorder g_get_segment_file_latency("doris_download_binlog", "get_segment_file");
bvar::LatencyRecorder g_get_segment_index_file_latency("doris_download_binlog",
"get_segment_index_file");
bvar::LatencyRecorder g_get_rowset_meta_latency("doris_download_binlog", "get_rowset_meta");

// get http param, if no value throw exception
const auto& get_http_param(HttpRequest* req, const std::string& param_name) {
const auto& param = req->param(param_name);
Expand Down Expand Up @@ -143,6 +150,7 @@ void handle_get_segment_index_file(HttpRequest* req,
const auto& rowset_id = get_http_param(req, kRowsetIdParameter);
const auto& segment_index = get_http_param(req, kSegmentIndexParameter);
const auto& segment_index_id = req->param(kSegmentIndexIdParameter);
auto segment_file_path = tablet->get_segment_filepath(rowset_id, segment_index);
segment_index_file_path =
tablet->get_segment_index_filepath(rowset_id, segment_index, segment_index_id);
is_acquire_md5 = !req->param(kAcquireMD5Parameter).empty();
Expand Down Expand Up @@ -218,14 +226,20 @@ void DownloadBinlogAction::handle(HttpRequest* req) {
const std::string& method = req->param(kMethodParameter);

// Step 3: dispatch
MonotonicStopWatch watch;
watch.start();
if (method == "get_binlog_info") {
handle_get_binlog_info(req);
g_get_binlog_info_latency << watch.elapsed_time_microseconds();
} else if (method == "get_segment_file") {
handle_get_segment_file(req, _rate_limit_group.get());
g_get_segment_file_latency << watch.elapsed_time_microseconds();
} else if (method == "get_segment_index_file") {
handle_get_segment_index_file(req, _rate_limit_group.get());
g_get_segment_index_file_latency << watch.elapsed_time_microseconds();
} else if (method == "get_rowset_meta") {
handle_get_rowset_meta(req);
g_get_rowset_meta_latency << watch.elapsed_time_microseconds();
} else {
auto error_msg = fmt::format("invalid method: {}", method);
LOG(WARNING) << error_msg;
Expand Down
6 changes: 4 additions & 2 deletions be/src/service/backend_service.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,10 @@ class TTransportException;
} // namespace apache

namespace doris {

namespace {

bvar::LatencyRecorder g_ingest_binlog_latency("doris_backend_service", "ingest_binlog");

struct IngestBinlogArg {
int64_t txn_id;
int64_t partition_id;
Expand Down Expand Up @@ -115,7 +116,8 @@ void _ingest_binlog(IngestBinlogArg* arg) {
std::vector<std::string> download_success_files;
Defer defer {[=, &tstatus, ingest_binlog_tstatus = arg->tstatus, &watch, &total_download_bytes,
&total_download_files]() {
auto elapsed_time_ms = static_cast<int64_t>(watch.elapsed_time() / 1000000);
g_ingest_binlog_latency << watch.elapsed_time_microseconds();
auto elapsed_time_ms = static_cast<int64_t>(watch.elapsed_time_milliseconds());
double copy_rate = 0.0;
if (elapsed_time_ms > 0) {
copy_rate = total_download_bytes / ((double)elapsed_time_ms) / 1000;
Expand Down
14 changes: 14 additions & 0 deletions be/src/util/stopwatch.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,20 @@ class CustomStopWatch {
(end.tv_nsec - _start.tv_nsec);
}

// Return time in microseconds
uint64_t elapsed_time_microseconds() const { return elapsed_time() / 1000; }

// Return time in milliseconds
uint64_t elapsed_time_milliseconds() const { return elapsed_time() / 1000 / 1000; }

// Returns time in nanosecond.
int64_t elapsed_time_seconds(timespec end) const {
if (!_running) {
return _total_time / 1000L / 1000L / 1000L;
}
return end.tv_sec - _start.tv_sec;
}

private:
timespec _start;
uint64_t _total_time; // in nanosec
Expand Down
Loading