Skip to content

Refact reader branch #64922

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Gabriel39 merged 2 commits into from
Jun 29, 2026
Merged

Refact reader branch #64922

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4ec377c
[feature](be) Add Remote Doris reader to file scanner v2
Gabriel39 Jun 26, 2026
77fa4e7
[fix](be) Build Remote Doris complex file schema children
Gabriel39 Jun 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions be/src/exec/scan/file_scanner_v2.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
#include "format_v2/table/hudi_reader.h"
#include "format_v2/table/iceberg_reader.h"
#include "format_v2/table/paimon_reader.h"
#include "format_v2/table/remote_doris_reader.h"
#include "format_v2/table_reader.h"
#include "io/fs/file_meta_cache.h"
#include "io/io_common.h"
Expand Down Expand Up @@ -90,6 +91,10 @@ bool is_supported_table_format(const TFileRangeDesc& range) {
table_format == "iceberg" || table_format == "paimon" || table_format == "hudi";
}

bool is_supported_arrow_table_format(const TFileRangeDesc& range) {
return table_format_name(range) == "remote_doris";
}

bool is_supported_jni_table_format(const TFileRangeDesc& range) {
const auto table_format = table_format_name(range);
if (table_format == "paimon") {
Expand Down Expand Up @@ -222,6 +227,8 @@ bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFile
const auto format_type = get_range_format_type(params, range);
if (format_type == TFileFormatType::FORMAT_PARQUET) {
return is_supported_table_format(range);
} else if (format_type == TFileFormatType::FORMAT_ARROW) {
return is_supported_arrow_table_format(range);
} else if (format_type == TFileFormatType::FORMAT_JNI) {
return is_supported_jni_table_format(range);
} else if (is_csv_format(format_type) || is_text_format(format_type) ||
Expand Down Expand Up @@ -379,6 +386,8 @@ Status FileScannerV2::_create_table_reader_for_format(
*reader = std::make_unique<format::max_compute::MaxComputeJniReader>(mc_desc);
} else if (table_format == "trino_connector") {
*reader = std::make_unique<format::trino_connector::TrinoConnectorJniReader>();
} else if (table_format == "remote_doris") {
*reader = std::make_unique<format::remote_doris::RemoteDorisReader>();
} else {
return Status::NotSupported("FileScannerV2 does not support table format {}", table_format);
}
Expand Down Expand Up @@ -622,6 +631,9 @@ Status FileScannerV2::_to_file_format(TFileFormatType::type format_type,
case TFileFormatType::FORMAT_NATIVE:
*file_format = format::FileFormat::NATIVE;
return Status::OK();
case TFileFormatType::FORMAT_ARROW:
*file_format = format::FileFormat::ARROW;
return Status::OK();
default:
return Status::NotSupported("FileScannerV2 does not support file format {}",
to_string(format_type));
Expand Down
1 change: 1 addition & 0 deletions be/src/format_v2/file_reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ enum class FileFormat {
TEXT,
JNI,
NATIVE,
ARROW,
};

// 通用文件层 scan 请求。
Expand Down
Loading
Loading