diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index 358663b65d0dad..a26448c90e2ce3 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -24,6 +24,8 @@ #include #include +#include "bvar/latency_recorder.h" +#include "bvar/reducer.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/logging.h" #include "common/sync_point.h" @@ -33,6 +35,13 @@ #include "util/doris_metrics.h" namespace doris::io { + +bvar::Adder hdfs_bytes_read_total("hdfs_file_reader", "bytes_read"); +bvar::LatencyRecorder hdfs_bytes_per_read("hdfs_file_reader", "bytes_per_read"); // also QPS +bvar::PerSecond> hdfs_read_througthput("hdfs_file_reader", + "hdfs_read_throughput", + &hdfs_bytes_read_total); + namespace { Result get_file(const hdfsFS& fs, const Path& file, int64_t mtime, @@ -148,6 +157,8 @@ Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_r has_read += loop_read; } *bytes_read = has_read; + hdfs_bytes_read_total << *bytes_read; + hdfs_bytes_per_read << *bytes_read; return Status::OK(); } @@ -206,6 +217,8 @@ Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_r has_read += loop_read; } *bytes_read = has_read; + hdfs_bytes_read_total << *bytes_read; + hdfs_bytes_per_read << *bytes_read; return Status::OK(); } #endif diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 2bd40fbbf43861..68acbf47eb18a1 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,9 @@ bvar::Adder s3_file_reader_read_counter("s3_file_reader", "read_at"); bvar::Adder s3_file_reader_total("s3_file_reader", "total_num"); bvar::Adder s3_bytes_read_total("s3_file_reader", "bytes_read"); bvar::Adder s3_file_being_read("s3_file_reader", "file_being_read"); +bvar::LatencyRecorder s3_bytes_per_read("s3_file_reader", "bytes_per_read"); // also QPS +bvar::PerSecond> s3_read_througthput("s3_file_reader", "s3_read_throughput", + &s3_bytes_read_total); Result S3FileReader::create(std::shared_ptr client, std::string bucket, std::string key, @@ -125,6 +129,7 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea _path.native(), *bytes_read, bytes_req); } s3_bytes_read_total << *bytes_read; + s3_bytes_per_read << *bytes_read; s3_file_reader_read_counter << 1; DorisMetrics::instance()->s3_bytes_read_total->increment(*bytes_read); return Status::OK();